Merge pull request #207 from openstack-k8s-operators/kuttl-autoscaling

KUTTL autoscaling

Makefile

@@ -325,3 +325,45 @@ run-with-webhook: export HEALTH_PORT?=8081
 run-with-webhook: manifests generate fmt vet ## Run a controller from your host.
 	/bin/bash hack/configure_local_webhook.sh
 	go run ./main.go -metrics-bind-address ":$(METRICS_PORT)" -health-probe-bind-address ":$(HEALTH_PORT)"
+
+.PHONY: kuttl-install
+kuttl-install:
+	./hack/install-krew.sh
+	export PATH="$${KREW_ROOT:-$$HOME/.krew}/bin:$$PATH" && kubectl krew install kuttl
+	echo "Place 'export PATH=$${KREW_ROOT:-$$HOME/.krew}/bin:$$PATH' to your ~/.bashrc"
+
+# TODO(mmagr): change this to default once there is default suite
+KUTTL_SUITE ?= autoscaling
+KUTTL_NAMESPACE ?= telemetry-kuttl-$(KUTTL_SUITE)
+KUTTL_SUITE_DIR ?= tests/kuttl/suites/$(KUTTL_SUITE)
+
+.PHONY: kuttl-test-prep
+kuttl-test-prep:
+	oc apply -k $(KUTTL_SUITE_DIR)/deps/ --timeout=120s
+	oc wait -n $(KUTTL_NAMESPACE) openstackcontrolplane openstack --for condition=Ready --timeout=300s
+	oc apply -f $(KUTTL_SUITE_DIR)/deps/rhobs.yaml
+	$(until oc api-resources -n $(KUTTL_NAMESPACE) | grep -q rhobs; do sleep 2; done)
+
+.PHONY: kuttl-test-run
+kuttl-test-run:
+	oc kuttl test --v 1 --start-kind=false --config $(KUTTL_SUITE_DIR)/config.yaml
+
+.PHONY: kuttl-test
+kuttl-test: kuttl-test-prep kuttl-test-run
+
+.PHONY: kuttl-test-cleanup
+kuttl-test-cleanup:
+	# only cleanup if the $(KUTTL_NAMESPACE) exists
+	$(eval namespace_exists=$(shell oc get namespace $(KUTTL_NAMESPACE) --ignore-not-found -o name))
+	# We need to order the deletion. Simply deleting the namespace will
+	# result in errors in mariadb- and keystone-operator and then
+	# finalizer removal get stuck blocking the namespace deletion.
+	if [ "${namespace_exists}" != "" ]; then \
+		if [ "$(KUTTL_SUITE)" == "autoscaling" ]; then \
+			oc delete --wait=true --all=true -n $(KUTTL_NAMESPACE) --timeout=120s Autoscaling; \
+		fi; \
+		oc delete --wait=true --all=true -n $(KUTTL_NAMESPACE) --timeout=120s OpenStackControlPlane; \
+		oc delete --wait=true namespace $(KUTTL_NAMESPACE); \
+	else \
+		echo "Namespce already cleaned up. Nothing to do"; \
+	fi

hack/install-krew.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# taken from https://github.com/openstack-k8s-operators/nova-operator/blob/main/hack/install-krew.sh
+(
+    set -x
+    tdir="$(mktemp -d)"
+    pushd $tdir &&
+    OS="$(uname | tr '[:upper:]' '[:lower:]')" &&
+    ARCH="$(uname -m | sed -e 's/x86_64/amd64/' -e 's/\(arm\)\(64\)\?.*/\1\2/' -e 's/aarch64$/arm64/')" &&
+    KREW="krew-${OS}_${ARCH}" &&
+    curl -fsSLO "https://github.com/kubernetes-sigs/krew/releases/latest/download/${KREW}.tar.gz" &&
+    tar zxvf "${KREW}.tar.gz" &&
+    ./"${KREW}" install krew
+    popd && rm -fr $tdir
+)

tests/kuttl/suites/autoscaling/config.yaml

@@ -0,0 +1,14 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestSuite
+reportFormat: JSON
+reportName: kuttl-autoscaling-results
+namespace: telemetry-kuttl-autoscaling
+# we could set this lower, but the initial image pull can take a while
+timeout: 300
+parallel: 1
+skipDelete: true
+testDirs:
+  - tests/kuttl/suites/autoscaling/
+suppress:
+  - events
+artifactsDir: tests/kuttl/suites/autoscaling/output

tests/kuttl/suites/autoscaling/deps/OpenStackControlPlane.yaml

@@ -0,0 +1,35 @@
+apiVersion: core.openstack.org/v1beta1
+kind: OpenStackControlPlane
+metadata:
+  name: openstack
+spec:
+  storageClass: "crc-csi-hostpath-provisioner"
+  keystone:
+    template:
+      databaseInstance: openstack
+      secret: osp-secret
+  ironic:
+    enabled: false
+    template:
+      ironicConductors: []
+  manila:
+    enabled: false
+    template:
+      manilaShares: {}
+  horizon:
+    enabled: false
+  nova:
+    enabled: false
+  placement:
+    template:
+      databaseInstance: openstack
+      secret: osp-secret
+  heat:
+    enabled: true
+    template:
+      databaseInstance: openstack
+      heatAPI:
+        replicas: 1
+      heatEngine:
+        replicas: 1
+      secret: osp-secret

tests/kuttl/suites/autoscaling/deps/infra.yaml

@@ -0,0 +1,41 @@
+apiVersion: core.openstack.org/v1beta1
+kind: OpenStackControlPlane
+metadata:
+  name: openstack
+spec:
+  mariadb:
+    enabled: false
+    templates:
+      openstack:
+        replicas: 0
+      openstack-cell1:
+        replicas: 0
+  galera:
+    enabled: true
+    templates:
+      openstack:
+        replicas: 1
+        storageRequest: 500M
+      openstack-cell1:
+        replicas: 1
+        storageRequest: 500M
+        secret: osp-secret
+    secret: osp-secret
+  rabbitmq:
+    templates:
+      rabbitmq:
+        replicas: 1
+      rabbitmq-cell1:
+        replicas: 1
+  memcached:
+    templates:
+      memcached:
+        replicas: 1
+  ovn:
+    enabled: false
+    template:
+      ovnController:
+        external-ids:
+          ovn-encap-type: geneve
+  ovs:
+    enabled: false

tests/kuttl/suites/autoscaling/deps/kustomization.yaml

@@ -0,0 +1,48 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: telemetry-kuttl-autoscaling
+
+secretGenerator:
+- literals:
+  - AdminPassword=password
+  - DbRootPassword=password
+  - DatabasePassword=password
+  - KeystoneDatabasePassword=password
+  - PlacementPassword=password
+  - PlacementDatabasePassword=password
+  - GlancePassword=password
+  - GlanceDatabasePassword=password
+  - NeutronPassword=password
+  - NeutronDatabasePassword=password
+  - NovaPassword=password
+  - NovaAPIDatabasePassword=password
+  - NovaCell0DatabasePassword=password
+  - NovaCell1DatabasePassword=password
+  - AodhPassword=password
+  - AodhDatabasePassword=password
+  - CeilometerPassword=password
+  - CeilometerDatabasePassword=password
+  - HeatPassword=password
+  - HeatDatabasePassword=password
+  - HeatAuthEncryptionKey=66699966699966600666999666999666
+  - MetadataSecret=42
+  name: osp-secret
+generatorOptions:
+  disableNameSuffixHash: true
+  labels:
+    type: osp-secret
+
+resources:
+- namespace.yaml
+- OpenStackControlPlane.yaml
+
+patches:
+- patch: |-
+    apiVersion: core.openstack.org/v1beta1
+    kind: OpenStackControlPlane
+    metadata:
+      name: openstack
+    spec:
+      secret: osp-secret
+- path: infra.yaml
+- path: telemetry.yaml

tests/kuttl/suites/autoscaling/deps/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: telemetry-kuttl

tests/kuttl/suites/autoscaling/deps/rhobs.yaml

@@ -0,0 +1,33 @@
+apiVersion: operators.coreos.com/v1alpha1
+kind: CatalogSource
+metadata:
+  annotations:
+  name: observability-operator
+  namespace: openshift-marketplace
+spec:
+  displayName: Observability Operator - Test
+  icon:
+    base64data: ""
+    mediatype: ""
+  image: quay.io/rhobs/observability-operator-catalog:latest
+  publisher: Sunil Thaha
+  sourceType: grpc
+  grpcPodConfig:
+    securityContextConfig: restricted
+  updateStrategy:
+    registryPoll:
+      interval: 1m0s
+---
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  labels:
+    operators.coreos.com/observability-operator.openshift-operators: ""
+  name: observability-operator
+  namespace: openshift-operators
+spec:
+  channel: development
+  installPlanApproval: Automatic
+  name: observability-operator
+  source: observability-operator
+  sourceNamespace: openshift-marketplace

tests/kuttl/suites/autoscaling/deps/telemetry.yaml

@@ -0,0 +1,7 @@
+apiVersion: core.openstack.org/v1beta1
+kind: OpenStackControlPlane
+metadata:
+  name: openstack
+spec:
+  telemetry:
+    enabled: false

tests/kuttl/suites/autoscaling/tests/00-assert.yaml

@@ -0,0 +1,101 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    prometheus: telemetry-kuttl-prometheus
+  name: prometheus-telemetry-kuttl-prometheus-0
+status:
+  containerStatuses:
+  - image: quay.io/rhobs/obo-prometheus-config-reloader:v0.68.0-rhobs1
+    name: config-reloader
+    ready: true
+    started: true
+  - image: quay.io/prometheus/prometheus:v2.47.0
+    name: prometheus
+    ready: true
+    started: true
+  - image: quay.io/thanos/thanos:v0.24.0
+    name: thanos-sidecar
+    ready: true
+    started: true
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: telemetry-kuttl-prometheus-prometheus
+  namespace: telemetry-kuttl-autoscaling
+  ownerReferences:
+    - kind: MonitoringStack
+      name: telemetry-kuttl-prometheus
+spec:
+  ports:
+  - name: web
+    port: 9090
+    protocol: TCP
+    targetPort: 9090
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    service: aodh
+    endpoint: internal
+  name: aodh-internal
+  namespace: telemetry-kuttl-autoscaling
+  ownerReferences:
+  - kind: Autoscaling
+    name: telemetry-kuttl
+spec:
+  ports:
+  - name: aodh-internal
+    port: 8042
+    protocol: TCP
+    targetPort: 8042
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    service: aodh
+    endpoint: public
+  name: aodh-public
+  namespace: telemetry-kuttl-autoscaling
+  ownerReferences:
+  - kind: Autoscaling
+    name: telemetry-kuttl
+spec:
+  ports:
+  - name: aodh-public
+    port: 8042
+    protocol: TCP
+    targetPort: 8042
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    service: aodh
+  name: aodh
+  namespace: telemetry-kuttl-autoscaling
+  ownerReferences:
+  - kind: Autoscaling
+    name: telemetry-kuttl
+spec:
+  template:
+    spec:
+      containers:
+      - image: quay.io/mmagr/openstack-aodh-api:current-podified
+        name: aodh-api
+      - image: quay.io/mmagr/openstack-aodh-evaluator:current-podified
+        name: aodh-evaluator
+      - image: quay.io/mmagr/openstack-aodh-notifier:current-podified
+        name: aodh-notifier
+      - image: quay.io/mmagr/openstack-aodh-listener:current-podified
+        name: aodh-listener
+      volumes:
+      - name: scripts
+        secret:
+          secretName: aodh-scripts
+      - name: config-data
+        secret:
+          secretName: aodh-config-data

tests/kuttl/suites/autoscaling/tests/00-deploy.yaml

@@ -0,0 +1,19 @@
+apiVersion: telemetry.openstack.org/v1beta1
+kind: Autoscaling
+metadata:
+  name: telemetry-kuttl
+spec:
+  enabled: true
+  prometheus:
+    deployPrometheus: true
+  aodh:
+    secret: osp-secret
+    apiImage: "quay.io/mmagr/openstack-aodh-api:current-podified"
+    evaluatorImage: "quay.io/mmagr/openstack-aodh-evaluator:current-podified"
+    notifierImage: "quay.io/mmagr/openstack-aodh-notifier:current-podified"
+    listenerImage: "quay.io/mmagr/openstack-aodh-listener:current-podified"
+    passwordSelectors:
+    databaseUser: aodh
+    databaseInstance: openstack
+    memcachedInstance: memcached
+  heatInstance: heat