From 2461b4e1e8b24f87ed0db94c5605dfda3c511cbd Mon Sep 17 00:00:00 2001 From: Reto Lehmann Date: Fri, 19 Jul 2024 09:07:00 +0200 Subject: [PATCH 01/10] Use OSSM 3.x as Service Mesh --- hack/lib/certmanager.bash | 5 +- hack/lib/mesh.bash | 71 ++++++++-------- .../serving-allow-traffic-to-activator.yaml | 2 +- .../lib/mesh_resources/destination-rules.yaml | 13 +++ hack/lib/mesh_resources/gateway-deploy.yaml | 50 +++++++++++ hack/lib/mesh_resources/gateway.yaml | 79 +++++++++++------ hack/lib/mesh_resources/istio-cni.yaml | 7 ++ hack/lib/mesh_resources/istio.yaml | 17 ++++ hack/lib/mesh_resources/namespace.yaml | 6 -- .../network-policy-monitoring.yaml | 13 --- .../peer-authentication-mesh-mtls.yaml | 9 ++ hack/lib/mesh_resources/smcp.yaml | 42 --------- hack/lib/mesh_resources/smmr.yaml | 19 ----- hack/lib/mesh_resources/subscription.yaml | 10 ++- hack/lib/serverless.bash | 85 ++++++++++--------- hack/lib/strimzi.bash | 2 +- test/e2e-tests.sh | 23 ++--- test/serving.bash | 10 +-- test/servinge2e/kourier/servicemesh_test.go | 6 ++ .../servicemesh/multitenant_test.go | 7 +- test/upstream-e2e-tests.sh | 18 ++-- 21 files changed, 270 insertions(+), 224 deletions(-) create mode 100644 hack/lib/mesh_resources/gateway-deploy.yaml create mode 100644 hack/lib/mesh_resources/istio-cni.yaml create mode 100644 hack/lib/mesh_resources/istio.yaml delete mode 100644 hack/lib/mesh_resources/network-policy-monitoring.yaml create mode 100644 hack/lib/mesh_resources/peer-authentication-mesh-mtls.yaml delete mode 100644 hack/lib/mesh_resources/smcp.yaml delete mode 100644 hack/lib/mesh_resources/smmr.yaml diff --git a/hack/lib/certmanager.bash b/hack/lib/certmanager.bash index c30a3ca202..e540e7ae3e 100644 --- a/hack/lib/certmanager.bash +++ b/hack/lib/certmanager.bash @@ -31,9 +31,6 @@ function deploy_certmanager_operator { oc apply -n "${deployment_namespace}" -f "${certmanager_resources_dir}"/serving-ca-certificate.yaml || return $? sync_trust_bundle "knative-selfsigned-ca" "knative-serving" "knative-serving-ingress" || return $? - if [[ $MESH == "true" ]]; then - sync_trust_bundle "knative-selfsigned-ca" "istio-system" || return $? - fi # eventing resources oc apply -f "${certmanager_resources_dir}"/selfsigned-issuer.yaml || return $? @@ -67,7 +64,7 @@ function sync_trust_bundle { for ns in "${namespaces[@]}"; do echo "Syncing trust-bundle for namespace: ${ns}" - oc create namespace "${ns}" --dry-run=client -o yaml | oc apply -f - + oc get namespace "${ns}" || oc create namespace "${ns}" oc label namespace "${ns}" knative.openshift.io/part-of="openshift-serverless" --overwrite oc create configmap -n "${ns}" knative-ca-bundle --from-file=tls.crt --from-file=ca.crt \ --dry-run=client -o yaml | kubectl apply -n "${ns}" -f - || return $? diff --git a/hack/lib/mesh.bash b/hack/lib/mesh.bash index 92400ba905..ee5beebe53 100644 --- a/hack/lib/mesh.bash +++ b/hack/lib/mesh.bash @@ -22,36 +22,26 @@ function deploy_servicemesh_operators { fi logger.info "Waiting until service mesh operators are available" - timeout 600 "[[ \$(oc get deploy -n openshift-operators istio-operator --no-headers | wc -l) != 1 ]]" || return 1 - oc wait --for=condition=Available deployment istio-operator --timeout=300s -n openshift-operators || return $? + timeout 600 "[[ \$(oc get deploy -n openshift-operators sail-operator --no-headers | wc -l) != 1 ]]" || return 1 + oc wait --for=condition=Available deployment sail-operator --timeout=300s -n openshift-operators || return $? } function undeploy_servicemesh_operators { logger.info "Deleting service mesh subscriptions" - oc delete subscriptions.operators.coreos.com -n openshift-operators servicemeshoperator kiali-ossm jaeger-product --ignore-not-found + oc delete subscriptions.operators.coreos.com -n openshift-operators sailoperator --ignore-not-found logger.info 'Deleting ClusterServiceVersion' for csv in $(set +o pipefail && oc get csv -n openshift-operators --no-headers 2>/dev/null \ - | grep 'servicemeshoperator\|jaeger\|kiali' | cut -f1 -d' '); do + | grep 'sailoperator' | cut -f1 -d' '); do oc delete csv -n openshift-operators "${csv}" done logger.info 'Ensure no operators present' - timeout 600 "[[ \$(oc get deployments -n openshift-operators -oname | grep -c 'servicemeshoperator\|jaeger\|kiali') != 0 ]]" - - logger.info "Deleting service mesh istio nodes" - oc delete --ignore-not-found=true daemonset.apps/istio-node -n openshift-operators - oc delete --ignore-not-found=true service/maistra-admission-controller -n openshift-operators + timeout 600 "[[ \$(oc get deployments -n openshift-operators -oname | grep -c 'sail-operator') != 0 ]]" logger.info "Deleting service mesh webhooks and rbac resources" - oc delete --ignore-not-found=true validatingwebhookconfiguration openshift-operators.servicemesh-resources.maistra.io - oc delete --ignore-not-found=true mutatingwebhookconfigurations openshift-operators.servicemesh-resources.maistra.io - oc delete --ignore-not-found=true clusterrole istio-admin istio-cni istio-edit istio-view - oc delete --ignore-not-found=true clusterrolebinding istio-cn + oc delete --ignore-not-found=true clusterrole istio-admin istio-edit istio-view logger.info 'Ensure not CRDs left' - if [[ ! $(oc get crd -oname | grep -c 'maistra.io') -eq 0 ]]; then - oc get crd -oname | grep 'maistra.io' | xargs oc delete --timeout=60s - fi if [[ ! $(oc get crd -oname | grep -c 'istio') -eq 0 ]]; then oc get crd -oname | grep 'istio' | xargs oc delete --timeout=60s fi @@ -59,28 +49,36 @@ function undeploy_servicemesh_operators { } function deploy_servicemeshcontrolplane { - logger.info "Installing ServiceMeshControlPlane in namespace istio-system" + logger.info "Installing istiod in namespace istio-system" oc get ns istio-system || oc create namespace istio-system + oc get ns istio-cni || oc create namespace istio-cni - # Make sure servicemeshcontrolplanes.maistra.io is available. - timeout 120 "[[ \$(oc get crd servicemeshcontrolplanes.maistra.io --no-headers | wc -l) != 1 ]]" || return 1 - oc wait --for=condition=Established crd servicemeshcontrolplanes.maistra.io + # Make sure istios.operator.istio.io is available. + timeout 120 "[[ \$(oc get crd istios.operator.istio.io --no-headers | wc -l) != 1 ]]" || return 1 + oc wait --for=condition=Established crd istios.operator.istio.io - # creating smcp often fails due to webhook error - timeout 120 "[[ \$(oc apply -f ${resources_dir}/smcp.yaml | oc get smcp -n istio-system basic --no-headers | wc -l) != 1 ]]" || return 1 + timeout 120 "[[ \$(oc apply -f ${resources_dir}/istio.yaml | oc get istios -n istio-system default --no-headers | wc -l) != 1 ]]" || return 1 + timeout 120 "[[ \$(oc apply -f ${resources_dir}/istio-cni.yaml | oc get istiocnis -n default default --no-headers | wc -l) != 1 ]]" || return 1 - if [[ $(oc get infrastructure cluster -ojsonpath='{.status.platformStatus.aws.resourceTags[?(@.key=="red-hat-clustertype")].value}') = rosa ]]; then - logger.info "ThirdParty tokens required when using ROSA cluster" - enable_smcp_third_party_token - fi +# TODO: CHECK ME for OSSM3 +# if [[ $(oc get infrastructure cluster -ojsonpath='{.status.platformStatus.aws.resourceTags[?(@.key=="red-hat-clustertype")].value}') = rosa ]]; then +# logger.info "ThirdParty tokens required when using ROSA cluster" +# enable_smcp_third_party_token +# fi + + oc wait --timeout=180s --for=condition=Ready istios -n istio-system default || oc get istios -n istio-system default -o yaml + oc wait --timeout=180s --for=condition=Ready istiocnis -n default default || oc get istiocnis -n default default -o yaml - oc wait --timeout=180s --for=condition=Ready smcp -n istio-system basic || oc get smcp -n istio-system basic -o yaml + # make sure istiod + cni pods are up before continuing + oc wait deploy --all --timeout=600s --for=condition=Available -n istio-system + oc rollout status daemonset -n istio-cni --timeout 600s } function undeploy_servicemeshcontrolplane { - logger.info "Deleting ServiceMeshControlPlane" - oc delete smcp -n istio-system basic --ignore-not-found || return $? + logger.info "Deleting istiod" + oc delete istios default -n istio-system --ignore-not-found || return $? + oc delete istiocnis default --ignore-not-found || return $? } function deploy_gateways { @@ -109,7 +107,9 @@ function deploy_gateways { -in "${out_dir}"/wildcard.csr \ -out "${out_dir}"/wildcard.crt - oc create -n istio-system secret tls wildcard-certs \ + oc apply -f "${resources_dir}"/namespace.yaml || return $? + + oc create -n knative-serving-ingress secret tls wildcard-certs \ --key="${out_dir}"/wildcard.key \ --cert="${out_dir}"/wildcard.crt --dry-run=client -o yaml | oc apply -f - @@ -119,32 +119,31 @@ function deploy_gateways { --key="${out_dir}"/wildcard.key \ --cert="${out_dir}"/wildcard.crt --dry-run=client -o yaml | oc apply -f - - oc apply -f "${resources_dir}"/namespace.yaml || return $? - oc apply -f "${resources_dir}"/smmr.yaml || return $? + oc apply -f "${resources_dir}"/gateway-deploy.yaml || return $? oc apply -f "${resources_dir}"/gateway.yaml || return $? oc apply -f "${resources_dir}"/authorization-policies/setup || return $? oc apply -f "${resources_dir}"/authorization-policies/helm || return $? oc apply -f "${resources_dir}"/destination-rules.yaml || return $? + oc apply -f "${resources_dir}"/peer-authentication-mesh-mtls.yaml || return $? oc apply -n "${EVENTING_NAMESPACE}" -f "${resources_dir}"/kafka-service-entry.yaml || return $? for ns in serverless-tests eventing-e2e0 eventing-e2e1 eventing-e2e2 eventing-e2e3 eventing-e2e4; do oc apply -n "$ns" -f "${resources_dir}"/kafka-service-entry.yaml || return $? done - oc apply -n "serverless-tests" -f "${resources_dir}"/network-policy-monitoring.yaml || return $? } function undeploy_gateways { - oc delete -n serverless-tests -f "${resources_dir}"/network-policy-monitoring.yaml --ignore-not-found || return $? for ns in serverless-tests eventing-e2e0 eventing-e2e1 eventing-e2e2 eventing-e2e3 eventing-e2e4; do oc delete -n "$ns" -f "${resources_dir}"/kafka-service-entry.yaml --ignore-not-found || return $? done + oc delete -f "${resources_dir}"/peer-authentication-mesh-mtls.yaml --ignore-not-found || return $? oc delete -f "${resources_dir}"/destination-rules.yaml --ignore-not-found || return $? oc delete -f "${resources_dir}"/authorization-policies/helm --ignore-not-found || return $? oc delete -f "${resources_dir}"/authorization-policies/setup --ignore-not-found || return $? oc delete -f "${resources_dir}"/gateway.yaml --ignore-not-found || return $? - oc delete -f "${resources_dir}"/smmr.yaml --ignore-not-found || return $? + oc delete -f "${resources_dir}"/gateway-deploy.yaml --ignore-not-found || return $? oc delete -n cert-manager secret ca-key-pair --ignore-not-found || return $? - oc delete -n istio-system secret wildcard-certs --ignore-not-found || return $? + oc delete -n knative-serving-ingress secret wildcard-certs --ignore-not-found || return $? } function enable_smcp_third_party_token { diff --git a/hack/lib/mesh_resources/authorization-policies/setup/serving-allow-traffic-to-activator.yaml b/hack/lib/mesh_resources/authorization-policies/setup/serving-allow-traffic-to-activator.yaml index e3df6cde55..092553b065 100644 --- a/hack/lib/mesh_resources/authorization-policies/setup/serving-allow-traffic-to-activator.yaml +++ b/hack/lib/mesh_resources/authorization-policies/setup/serving-allow-traffic-to-activator.yaml @@ -11,4 +11,4 @@ spec: rules: - from: - source: - namespaces: [ "knative-serving", "istio-system" ] + namespaces: [ "knative-serving", "knative-serving-ingress" ] diff --git a/hack/lib/mesh_resources/destination-rules.yaml b/hack/lib/mesh_resources/destination-rules.yaml index 96248783cd..8759ff0cdf 100644 --- a/hack/lib/mesh_resources/destination-rules.yaml +++ b/hack/lib/mesh_resources/destination-rules.yaml @@ -10,3 +10,16 @@ spec: tls: mode: DISABLE --- +# This DR forces mTLS to be used for DomainMappings which are routed from +# knative-gateway to knative-local-gateway in knative-serving-ingress +apiVersion: networking.istio.io/v1beta1 +kind: DestinationRule +metadata: + name: local-gateway-force-mtls + namespace: knative-serving-ingress +spec: + host: 'knative-local-gateway.knative-serving-ingress.svc.cluster.local' + trafficPolicy: + tls: + mode: ISTIO_MUTUAL +--- diff --git a/hack/lib/mesh_resources/gateway-deploy.yaml b/hack/lib/mesh_resources/gateway-deploy.yaml new file mode 100644 index 0000000000..deaf945fce --- /dev/null +++ b/hack/lib/mesh_resources/gateway-deploy.yaml @@ -0,0 +1,50 @@ +# based on OSSM 3.0 samples: https://raw.githubusercontent.com/maistra/istio-operator/maistra-3.0/chart/samples/ingress-gateway.yaml +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: knative-istio-ingressgateway + namespace: knative-serving-ingress +spec: + selector: + matchLabels: + knative: ingressgateway + template: + metadata: + annotations: + # Select the gateway injection template (rather than the default sidecar template) + inject.istio.io/templates: gateway + labels: + # Set a unique label for the gateway. This is required to ensure Gateways can select this workload + knative: ingressgateway + sidecar.istio.io/inject: "true" + spec: + containers: + - name: istio-proxy + image: auto # The image will automatically update each time the pod starts. + +--- +# Set up roles to allow reading credentials for TLS +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: istio-ingressgateway-sds + namespace: knative-serving-ingress +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "watch", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: istio-ingressgateway-sds + namespace: knative-serving-ingress +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: istio-ingressgateway-sds +subjects: + - kind: ServiceAccount + name: default +--- diff --git a/hack/lib/mesh_resources/gateway.yaml b/hack/lib/mesh_resources/gateway.yaml index a325a813a0..3206af0823 100644 --- a/hack/lib/mesh_resources/gateway.yaml +++ b/hack/lib/mesh_resources/gateway.yaml @@ -1,55 +1,82 @@ -apiVersion: networking.istio.io/v1alpha3 -kind: Gateway +########################################################### +# cluster external +########################################################### +apiVersion: v1 +kind: Service metadata: - name: knative-ingress-gateway - namespace: knative-serving + name: knative-istio-ingressgateway + namespace: knative-serving-ingress spec: + type: ClusterIP selector: knative: ingressgateway - servers: - - port: - number: 443 - name: https - protocol: HTTPS - hosts: - - "*" - tls: - mode: SIMPLE - credentialName: wildcard-certs + ports: + - name: http2 + port: 80 + targetPort: 8080 + - name: https + port: 443 + targetPort: 8443 --- -apiVersion: networking.istio.io/v1alpha3 +apiVersion: networking.istio.io/v1beta1 kind: Gateway metadata: - name: knative-local-gateway + name: knative-ingress-gateway namespace: knative-serving spec: selector: knative: ingressgateway servers: - - port: - number: 8081 + - hosts: + - '*' + port: name: https + number: 443 protocol: HTTPS tls: - mode: ISTIO_MUTUAL - hosts: - - "*" + credentialName: wildcard-certs + mode: SIMPLE --- +########################################################### +# cluster local +########################################################### apiVersion: v1 kind: Service metadata: - name: knative-local-gateway - namespace: istio-system labels: experimental.istio.io/disable-gateway-port-translation: "true" + name: knative-local-gateway + namespace: knative-serving-ingress spec: - type: ClusterIP - selector: - istio: ingressgateway ports: - name: http2 port: 80 + protocol: TCP targetPort: 8081 - name: https port: 443 + protocol: TCP targetPort: 8444 + selector: + knative: ingressgateway + type: ClusterIP +######################### +--- +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: knative-local-gateway + namespace: knative-serving +spec: + selector: + knative: ingressgateway + servers: + - hosts: + - "*" + port: + number: 8081 + name: https + protocol: HTTPS + tls: + mode: ISTIO_MUTUAL +--- diff --git a/hack/lib/mesh_resources/istio-cni.yaml b/hack/lib/mesh_resources/istio-cni.yaml new file mode 100644 index 0000000000..2c24bb5c47 --- /dev/null +++ b/hack/lib/mesh_resources/istio-cni.yaml @@ -0,0 +1,7 @@ +kind: IstioCNI +apiVersion: operator.istio.io/v1alpha1 +metadata: + name: default +spec: + namespace: istio-cni + version: v1.21.0 diff --git a/hack/lib/mesh_resources/istio.yaml b/hack/lib/mesh_resources/istio.yaml new file mode 100644 index 0000000000..b93edfb1da --- /dev/null +++ b/hack/lib/mesh_resources/istio.yaml @@ -0,0 +1,17 @@ +apiVersion: operator.istio.io/v1alpha1 +kind: Istio +metadata: + name: default # this is propagated to the revision and "istio-injection: enabled" only works with this name +spec: + values: + global: + proxy: + excludeInboundPorts: "8444,8022" # metrics, serving: wait-for-drain k8s pre-stop hook + meshConfig: + defaultConfig: + terminationDrainDuration: 35s # needed to make QP stop hook work + updateStrategy: + inactiveRevisionDeletionGracePeriodSeconds: 30 + type: InPlace + namespace: istio-system + version: v1.21.0 diff --git a/hack/lib/mesh_resources/namespace.yaml b/hack/lib/mesh_resources/namespace.yaml index fd264c0161..0406158ec0 100644 --- a/hack/lib/mesh_resources/namespace.yaml +++ b/hack/lib/mesh_resources/namespace.yaml @@ -17,22 +17,16 @@ apiVersion: v1 kind: Namespace metadata: name: serving-tests - labels: - maistra.io/member-of: istio-system # Workaround for OSSM-1397 --- apiVersion: v1 kind: Namespace metadata: name: serving-tests-alt - labels: - maistra.io/member-of: istio-system # Workaround for OSSM-1397 --- apiVersion: v1 kind: Namespace metadata: name: serverless-tests - labels: - maistra.io/member-of: istio-system # Workaround for OSSM-1397 --- # Additional namespaces for upgrade tests apiVersion: v1 diff --git a/hack/lib/mesh_resources/network-policy-monitoring.yaml b/hack/lib/mesh_resources/network-policy-monitoring.yaml deleted file mode 100644 index 26b664035a..0000000000 --- a/hack/lib/mesh_resources/network-policy-monitoring.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: allow-from-openshift-monitoring -spec: - ingress: - - from: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: openshift-monitoring - podSelector: {} - policyTypes: - - Ingress diff --git a/hack/lib/mesh_resources/peer-authentication-mesh-mtls.yaml b/hack/lib/mesh_resources/peer-authentication-mesh-mtls.yaml new file mode 100644 index 0000000000..37e38e044a --- /dev/null +++ b/hack/lib/mesh_resources/peer-authentication-mesh-mtls.yaml @@ -0,0 +1,9 @@ +# this forces the whole mesh to mTLS only +apiVersion: "security.istio.io/v1beta1" +kind: "PeerAuthentication" +metadata: + name: mesh-mtls + namespace: istio-system +spec: + mtls: + mode: STRICT \ No newline at end of file diff --git a/hack/lib/mesh_resources/smcp.yaml b/hack/lib/mesh_resources/smcp.yaml deleted file mode 100644 index 787ab4b8df..0000000000 --- a/hack/lib/mesh_resources/smcp.yaml +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: maistra.io/v2 -kind: ServiceMeshControlPlane -metadata: - name: basic - namespace: istio-system -spec: - profiles: - - default - security: - dataPlane: - mtls: true - techPreview: - meshConfig: - defaultConfig: - terminationDrainDuration: 35s - gateways: - ingress: - service: - metadata: - labels: - knative: ingressgateway - proxy: - networking: - trafficControl: - inbound: - excludedPorts: - - 8444 # metrics - - 8022 # serving: wait-for-drain k8s pre-stop hook - accessLogging: - file: - name: /dev/stdout - format: "{ \"authority\": \"%REQ(:AUTHORITY)%\", \"bytes_received\": %BYTES_RECEIVED%, \"bytes_sent\": %BYTES_SENT%, \"downstream_local_address\": \"%DOWNSTREAM_LOCAL_ADDRESS%\", \"downstream_peer_cert_v_end\": \"%DOWNSTREAM_PEER_CERT_V_END%\", \"downstream_peer_cert_v_start\": \"%DOWNSTREAM_PEER_CERT_V_START%\", \"downstream_remote_address\": \"%DOWNSTREAM_REMOTE_ADDRESS%\", \"downstream_tls_cipher\": \"%DOWNSTREAM_TLS_CIPHER%\", \"downstream_tls_version\": \"%DOWNSTREAM_TLS_VERSION%\", \"duration\": %DURATION%, \"hostname\": \"%HOSTNAME%\", \"istio_policy_status\": \"%DYNAMIC_METADATA(istio.mixer:status)%\", \"method\": \"%REQ(:METHOD)%\", \"path\": \"%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%\", \"protocol\": \"%PROTOCOL%\", \"request_duration\": %REQUEST_DURATION%, \"request_id\": \"%REQ(X-REQUEST-ID)%\", \"requested_server_name\": \"%REQUESTED_SERVER_NAME%\", \"response_code\": \"%RESPONSE_CODE%\", \"response_duration\": %RESPONSE_DURATION%, \"response_tx_duration\": %RESPONSE_TX_DURATION%, \"response_flags\": \"%RESPONSE_FLAGS%\", \"route_name\": \"%ROUTE_NAME%\", \"start_time\": \"%START_TIME%\", \"upstream_cluster\": \"%UPSTREAM_CLUSTER%\", \"upstream_host\": \"%UPSTREAM_HOST%\", \"upstream_local_address\": \"%UPSTREAM_LOCAL_ADDRESS%\", \"upstream_service_time\": %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%, \"upstream_transport_failure_reason\": \"%UPSTREAM_TRANSPORT_FAILURE_REASON%\", \"user_agent\": \"%REQ(USER-AGENT)%\", \"x_forwarded_for\": \"%REQ(X-FORWARDED-FOR)%\" } \n" - tracing: - type: None - addons: - grafana: - enabled: false - kiali: - enabled: false - prometheus: - enabled: false - version: v2.4 diff --git a/hack/lib/mesh_resources/smmr.yaml b/hack/lib/mesh_resources/smmr.yaml deleted file mode 100644 index 69638717d9..0000000000 --- a/hack/lib/mesh_resources/smmr.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: maistra.io/v1 -kind: ServiceMeshMemberRoll -metadata: - name: default - namespace: istio-system -spec: - members: - - knative-serving - - knative-eventing - - serving-tests - - serving-tests-alt - - serverless-tests - - eventing-e2e0 - - eventing-e2e1 - - eventing-e2e2 - - eventing-e2e3 - - eventing-e2e4 - - tenant-1 - - tenant-2 diff --git a/hack/lib/mesh_resources/subscription.yaml b/hack/lib/mesh_resources/subscription.yaml index 8e58f92fc5..68dfb17aab 100644 --- a/hack/lib/mesh_resources/subscription.yaml +++ b/hack/lib/mesh_resources/subscription.yaml @@ -1,11 +1,13 @@ apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: - name: servicemeshoperator + name: sailoperator namespace: openshift-operators spec: - channel: stable - name: servicemeshoperator + channel: 3.0-nightly installPlanApproval: Automatic - source: redhat-operators + name: sailoperator + source: community-operators sourceNamespace: openshift-marketplace + startingCSV: sailoperator.v3.0.0-nightly-2024-05-13 +--- diff --git a/hack/lib/serverless.bash b/hack/lib/serverless.bash index 1152627343..6706e80b35 100644 --- a/hack/lib/serverless.bash +++ b/hack/lib/serverless.bash @@ -203,20 +203,27 @@ function enable_istio { istio_patch="$(mktemp -t istio-XXXXX.yaml)" cat - << EOF > "${istio_patch}" +metadata: + annotations: + serverless.openshift.io/disable-istio-net-policies-generation: "true" spec: ingress: istio: enabled: true + config: + istio: # point these to our own specific gateways now + gateway.knative-serving.knative-ingress-gateway: knative-istio-ingressgateway.knative-serving-ingress.svc.cluster.local + local-gateway.knative-serving.knative-local-gateway: knative-local-gateway.knative-serving-ingress.svc.cluster.local deployments: - - labels: - sidecar.istio.io/inject: "true" - annotations: + - annotations: sidecar.istio.io/rewriteAppHTTPProbers: "true" - name: activator - - labels: + labels: sidecar.istio.io/inject: "true" - annotations: + name: activator + - annotations: sidecar.istio.io/rewriteAppHTTPProbers: "true" + labels: + sidecar.istio.io/inject: "true" name: autoscaler EOF @@ -241,29 +248,29 @@ spec: istio: "enabled" delivery-timeout: "enabled" workloads: - - labels: - sidecar.istio.io/inject: "true" - annotations: + - annotations: sidecar.istio.io/logLevel: "debug" sidecar.istio.io/rewriteAppHTTPProbers: "true" - name: pingsource-mt-adapter - - labels: + labels: sidecar.istio.io/inject: "true" - annotations: + name: pingsource-mt-adapter + - annotations: sidecar.istio.io/logLevel: "debug" sidecar.istio.io/rewriteAppHTTPProbers: "true" - name: mt-broker-ingress - - labels: + labels: sidecar.istio.io/inject: "true" - annotations: + name: mt-broker-ingress + - annotations: sidecar.istio.io/logLevel: "debug" sidecar.istio.io/rewriteAppHTTPProbers: "true" - name: mt-broker-filter - - labels: + labels: sidecar.istio.io/inject: "true" - annotations: + name: mt-broker-filter + - annotations: sidecar.istio.io/logLevel: "debug" sidecar.istio.io/rewriteAppHTTPProbers: "true" + labels: + sidecar.istio.io/inject: "true" name: imc-dispatcher EOF @@ -304,47 +311,47 @@ function enable_istio_eventing_kafka { cat - << EOF > "${istio_patch}" spec: workloads: - - labels: - sidecar.istio.io/inject: "true" - annotations: + - annotations: sidecar.istio.io/logLevel: "debug" sidecar.istio.io/rewriteAppHTTPProbers: "true" - name: kafka-broker-receiver - - labels: + labels: sidecar.istio.io/inject: "true" - annotations: + name: kafka-broker-receiver + - annotations: sidecar.istio.io/logLevel: "debug" sidecar.istio.io/rewriteAppHTTPProbers: "true" - name: kafka-broker-dispatcher - - labels: + labels: sidecar.istio.io/inject: "true" - annotations: + name: kafka-broker-dispatcher + - annotations: sidecar.istio.io/logLevel: "debug" sidecar.istio.io/rewriteAppHTTPProbers: "true" - name: kafka-channel-receiver - - labels: + labels: sidecar.istio.io/inject: "true" - annotations: + name: kafka-channel-receiver + - annotations: sidecar.istio.io/logLevel: "debug" sidecar.istio.io/rewriteAppHTTPProbers: "true" - name: kafka-channel-dispatcher - - labels: + labels: sidecar.istio.io/inject: "true" - annotations: + name: kafka-channel-dispatcher + - annotations: sidecar.istio.io/logLevel: "debug" sidecar.istio.io/rewriteAppHTTPProbers: "true" - name: kafka-sink-receiver - - labels: + labels: sidecar.istio.io/inject: "true" - annotations: + name: kafka-sink-receiver + - annotations: sidecar.istio.io/logLevel: "debug" sidecar.istio.io/rewriteAppHTTPProbers: "true" - name: kafka-source-dispatcher - - labels: + labels: sidecar.istio.io/inject: "true" - annotations: + name: kafka-source-dispatcher + - annotations: sidecar.istio.io/logLevel: "debug" sidecar.istio.io/rewriteAppHTTPProbers: "true" + labels: + sidecar.istio.io/inject: "true" name: kafka-controller EOF diff --git a/hack/lib/strimzi.bash b/hack/lib/strimzi.bash index 9622d66f1b..1d1eb8baa4 100644 --- a/hack/lib/strimzi.bash +++ b/hack/lib/strimzi.bash @@ -380,7 +380,7 @@ EOF function install_strimzi { logger.info "Strimzi install" - oc create namespace "${EVENTING_NAMESPACE}" --dry-run=client -o yaml | oc apply -f - + oc get namespace "${EVENTING_NAMESPACE}" || oc create namespace "${EVENTING_NAMESPACE}" install_strimzi_operator install_strimzi_cluster install_strimzi_users diff --git a/test/e2e-tests.sh b/test/e2e-tests.sh index d90b2c0696..f6dd2dc657 100755 --- a/test/e2e-tests.sh +++ b/test/e2e-tests.sh @@ -12,11 +12,6 @@ fi debugging.setup # both install and test dump_state.setup # test -if [[ $MESH == "true" ]]; then - # net-istio does not use knative-serving-ingress namespace. - export INGRESS_NAMESPACE="knative-serving" -fi - logger.success '🚀 Cluster prepared for testing.' trust_router_ca @@ -31,21 +26,21 @@ fi run_testselect serverless_operator_e2e_tests -if [[ $TEST_KNATIVE_KAFKA == true ]]; then - serverless_operator_kafka_e2e_tests -fi +#if [[ $TEST_KNATIVE_KAFKA == true ]]; then +# serverless_operator_kafka_e2e_tests +#fi [ -n "$OPENSHIFT_CI" ] && setup_quick_api_deprecation_alerts # Run Knative Serving & Eventing downstream E2E tests. downstream_serving_e2e_tests -downstream_eventing_e2e_tests -downstream_eventing_e2e_rekt_tests +#downstream_eventing_e2e_tests +#downstream_eventing_e2e_rekt_tests downstream_monitoring_e2e_tests -if [[ $TEST_KNATIVE_KAFKA == true ]]; then - downstream_knative_kafka_e2e_tests - downstream_knative_kafka_e2e_rekt_tests -fi +#if [[ $TEST_KNATIVE_KAFKA == true ]]; then +# downstream_knative_kafka_e2e_tests +# downstream_knative_kafka_e2e_rekt_tests +#fi [ -n "$OPENSHIFT_CI" ] && check_serverless_alerts diff --git a/test/serving.bash b/test/serving.bash index edb466804b..5bb9f0c743 100644 --- a/test/serving.bash +++ b/test/serving.bash @@ -22,18 +22,12 @@ function prepare_knative_serving_tests { # Create test resources (namespaces, configMaps, secrets) oc apply -f test/config/cluster-resources.yaml # Workaround for https://issues.redhat.com/browse/OSSM-1397 - if [[ $MESH == "true" ]]; then - oc label namespace serving-tests maistra.io/member-of=istio-system --overwrite - fi oc apply -f test/config/test-resources.yaml # Adding scc for anyuid to test TestShouldRunAsUserContainerDefault. oc adm policy add-scc-to-user anyuid -z default -n serving-tests - # Add networkpolicy to test namespace and label to serving namespaces for testing under the strict networkpolicy. - add_networkpolicy "serving-tests" - add_networkpolicy "serving-tests-alt" - export GATEWAY_OVERRIDE="kourier" - export GATEWAY_NAMESPACE_OVERRIDE="${INGRESS_NAMESPACE}" + export GATEWAY_OVERRIDE="kourier" + export GATEWAY_NAMESPACE_OVERRIDE="${INGRESS_NAMESPACE}" } function upstream_knative_serving_e2e_and_conformance_tests { diff --git a/test/servinge2e/kourier/servicemesh_test.go b/test/servinge2e/kourier/servicemesh_test.go index c452e90a9f..d6a167b524 100644 --- a/test/servinge2e/kourier/servicemesh_test.go +++ b/test/servinge2e/kourier/servicemesh_test.go @@ -102,6 +102,9 @@ func TestKsvcWithServiceMeshSidecar(t *testing.T) { // Requests go via gateway -> activator -> pod , by default // Verifies the activator can connect to the pod name: "sidecar-via-activator", + labels: map[string]string{ + servicemesh.IstioInjectKey: "true", + }, annotations: map[string]string{ autoscaling.TargetBurstCapacityKey: "-1", }, @@ -113,6 +116,9 @@ func TestKsvcWithServiceMeshSidecar(t *testing.T) { // Requests go via gateway -> pod ( activator should be skipped if burst capacity is disabled and there is at least 1 replica) // Verifies the gateway can connect to the pod directly name: "sidecar-without-activator", + labels: map[string]string{ + servicemesh.IstioInjectKey: "true", + }, annotations: map[string]string{ autoscaling.TargetBurstCapacityKey: "0", autoscaling.MinScaleAnnotationKey: "1", diff --git a/test/servinge2e/servicemesh/multitenant_test.go b/test/servinge2e/servicemesh/multitenant_test.go index 83a3fa406f..4790bbd17e 100644 --- a/test/servinge2e/servicemesh/multitenant_test.go +++ b/test/servinge2e/servicemesh/multitenant_test.go @@ -18,7 +18,7 @@ import ( const ( Tenant1 = "tenant-1" Tenant2 = "tenant-2" - LocalGatewayHost = "knative-local-gateway.istio-system.svc.cluster.local" + LocalGatewayHost = "knative-local-gateway.knative-serving-ingress.svc.cluster.local" ) var ExpectStatusForbidden = func(resp *spoof.Response) (bool, error) { @@ -117,13 +117,13 @@ func TestMultiTenancyWithServiceMesh(t *testing.T) { for _, tc := range tests { tc := tc - tc.annotations[IstioInjectKey] = "true" tc.annotations[IstioRewriteProbersKey] = "true" // Always use cluster-local service. tc.labels = map[string]string{ networking.VisibilityLabelKey: serving.VisibilityClusterLocal, } + t.Run(tc.name, func(t *testing.T) { t.Parallel() @@ -135,6 +135,9 @@ func TestMultiTenancyWithServiceMesh(t *testing.T) { ServingEnablePassthroughKey: "true", }, tc.annotations) service.ObjectMeta.Labels = tc.labels + service.Spec.Template.Labels = map[string]string{ + IstioInjectKey: "true", + } service = test.WithServiceReadyOrFail(ctx, service) diff --git a/test/upstream-e2e-tests.sh b/test/upstream-e2e-tests.sh index c9e5bb4395..b39f7270d9 100755 --- a/test/upstream-e2e-tests.sh +++ b/test/upstream-e2e-tests.sh @@ -25,9 +25,9 @@ run_testselect # Run upgrade tests if [[ $TEST_KNATIVE_UPGRADE == true ]]; then # Set KafkaChannel as default for upgrade tests. - if [[ $TEST_KNATIVE_KAFKA == "true" ]]; then - ensure_kafka_channel_default - fi +# if [[ $TEST_KNATIVE_KAFKA == "true" ]]; then +# ensure_kafka_channel_default +# fi run_rolling_upgrade_tests fi @@ -36,15 +36,15 @@ if [[ $TEST_KNATIVE_E2E == true ]]; then # TODO: Remove this when upstream tests can use in-cluster config. # See https://github.com/knative/eventing/issues/5996 (the same issue affects Eventing Kafka) ensure_kubeconfig - if [[ $TEST_KNATIVE_KAFKA_BROKER == true ]]; then - upstream_knative_eventing_kafka_broker_e2e - fi +# if [[ $TEST_KNATIVE_KAFKA_BROKER == true ]]; then +# upstream_knative_eventing_kafka_broker_e2e +# fi if [[ $TEST_KNATIVE_SERVING == true ]]; then upstream_knative_serving_e2e_and_conformance_tests fi - if [[ $TEST_KNATIVE_EVENTING == true ]]; then - upstream_knative_eventing_e2e - fi +# if [[ $TEST_KNATIVE_EVENTING == true ]]; then +# upstream_knative_eventing_e2e +# fi fi [ -n "$OPENSHIFT_CI" ] && check_serverless_alerts From 0671e0cb70e82b4b9ea66f0afaa6954154f5f128 Mon Sep 17 00:00:00 2001 From: Reto Lehmann Date: Mon, 30 Sep 2024 11:36:22 +0200 Subject: [PATCH 02/10] Add istio-inject label to all namespaces --- hack/lib/mesh_resources/namespace.yaml | 26 +++++++++ hack/lib/serverless.bash | 78 +++++++++++++------------- 2 files changed, 65 insertions(+), 39 deletions(-) diff --git a/hack/lib/mesh_resources/namespace.yaml b/hack/lib/mesh_resources/namespace.yaml index 0406158ec0..19f96d8dd2 100644 --- a/hack/lib/mesh_resources/namespace.yaml +++ b/hack/lib/mesh_resources/namespace.yaml @@ -2,64 +2,90 @@ apiVersion: v1 kind: Namespace metadata: name: knative-serving + labels: + istio-injection: enabled --- apiVersion: v1 kind: Namespace metadata: name: knative-serving-ingress + labels: + istio-injection: enabled --- apiVersion: v1 kind: Namespace metadata: name: knative-eventing + labels: + istio-injection: enabled --- apiVersion: v1 kind: Namespace metadata: name: serving-tests + labels: + istio-injection: enabled --- apiVersion: v1 kind: Namespace metadata: name: serving-tests-alt + labels: + istio-injection: enabled --- apiVersion: v1 kind: Namespace metadata: name: serverless-tests + labels: + istio-injection: enabled --- # Additional namespaces for upgrade tests apiVersion: v1 kind: Namespace metadata: name: eventing-e2e0 + labels: + istio-injection: enabled --- apiVersion: v1 kind: Namespace metadata: name: eventing-e2e1 + labels: + istio-injection: enabled --- apiVersion: v1 kind: Namespace metadata: name: eventing-e2e2 + labels: + istio-injection: enabled --- apiVersion: v1 kind: Namespace metadata: name: eventing-e2e3 + labels: + istio-injection: enabled --- apiVersion: v1 kind: Namespace metadata: name: eventing-e2e4 + labels: + istio-injection: enabled --- apiVersion: v1 kind: Namespace metadata: name: tenant-1 + labels: + istio-injection: enabled --- apiVersion: v1 kind: Namespace metadata: name: tenant-2 + labels: + istio-injection: enabled diff --git a/hack/lib/serverless.bash b/hack/lib/serverless.bash index 6706e80b35..4a0b99d375 100644 --- a/hack/lib/serverless.bash +++ b/hack/lib/serverless.bash @@ -215,15 +215,15 @@ spec: gateway.knative-serving.knative-ingress-gateway: knative-istio-ingressgateway.knative-serving-ingress.svc.cluster.local local-gateway.knative-serving.knative-local-gateway: knative-local-gateway.knative-serving-ingress.svc.cluster.local deployments: - - annotations: - sidecar.istio.io/rewriteAppHTTPProbers: "true" - labels: + - labels: sidecar.istio.io/inject: "true" - name: activator - - annotations: + annotations: sidecar.istio.io/rewriteAppHTTPProbers: "true" - labels: + name: activator + - labels: sidecar.istio.io/inject: "true" + annotations: + sidecar.istio.io/rewriteAppHTTPProbers: "true" name: autoscaler EOF @@ -248,29 +248,29 @@ spec: istio: "enabled" delivery-timeout: "enabled" workloads: - - annotations: + - labels: + sidecar.istio.io/inject: "true" + annotations: sidecar.istio.io/logLevel: "debug" sidecar.istio.io/rewriteAppHTTPProbers: "true" - labels: - sidecar.istio.io/inject: "true" name: pingsource-mt-adapter - - annotations: + - labels: + sidecar.istio.io/inject: "true" + annotations: sidecar.istio.io/logLevel: "debug" sidecar.istio.io/rewriteAppHTTPProbers: "true" - labels: - sidecar.istio.io/inject: "true" name: mt-broker-ingress - - annotations: + - labels: + sidecar.istio.io/inject: "true" + annotations: sidecar.istio.io/logLevel: "debug" sidecar.istio.io/rewriteAppHTTPProbers: "true" - labels: - sidecar.istio.io/inject: "true" name: mt-broker-filter - - annotations: + - labels: + sidecar.istio.io/inject: "true" + annotations: sidecar.istio.io/logLevel: "debug" sidecar.istio.io/rewriteAppHTTPProbers: "true" - labels: - sidecar.istio.io/inject: "true" name: imc-dispatcher EOF @@ -311,47 +311,47 @@ function enable_istio_eventing_kafka { cat - << EOF > "${istio_patch}" spec: workloads: - - annotations: + - labels: + sidecar.istio.io/inject: "true" + annotations: sidecar.istio.io/logLevel: "debug" sidecar.istio.io/rewriteAppHTTPProbers: "true" - labels: - sidecar.istio.io/inject: "true" name: kafka-broker-receiver - - annotations: + - labels: + sidecar.istio.io/inject: "true" + annotations: sidecar.istio.io/logLevel: "debug" sidecar.istio.io/rewriteAppHTTPProbers: "true" - labels: - sidecar.istio.io/inject: "true" name: kafka-broker-dispatcher - - annotations: + - labels: + sidecar.istio.io/inject: "true" + annotations: sidecar.istio.io/logLevel: "debug" sidecar.istio.io/rewriteAppHTTPProbers: "true" - labels: - sidecar.istio.io/inject: "true" name: kafka-channel-receiver - - annotations: + - labels: + sidecar.istio.io/inject: "true" + annotations: sidecar.istio.io/logLevel: "debug" sidecar.istio.io/rewriteAppHTTPProbers: "true" - labels: - sidecar.istio.io/inject: "true" name: kafka-channel-dispatcher - - annotations: + - labels: + sidecar.istio.io/inject: "true" + annotations: sidecar.istio.io/logLevel: "debug" sidecar.istio.io/rewriteAppHTTPProbers: "true" - labels: - sidecar.istio.io/inject: "true" name: kafka-sink-receiver - - annotations: + - labels: + sidecar.istio.io/inject: "true" + annotations: sidecar.istio.io/logLevel: "debug" sidecar.istio.io/rewriteAppHTTPProbers: "true" - labels: - sidecar.istio.io/inject: "true" name: kafka-source-dispatcher - - annotations: + - labels: + sidecar.istio.io/inject: "true" + annotations: sidecar.istio.io/logLevel: "debug" sidecar.istio.io/rewriteAppHTTPProbers: "true" - labels: - sidecar.istio.io/inject: "true" name: kafka-controller EOF From 4ed03a2653d23194021b9cb6399b9b5ba6a60bbc Mon Sep 17 00:00:00 2001 From: Reto Lehmann Date: Tue, 1 Oct 2024 13:12:32 +0200 Subject: [PATCH 03/10] Drop unused function add_networkpolicy --- test/lib.bash | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/test/lib.bash b/test/lib.bash index 9876c63d49..7ffbccbb4a 100644 --- a/test/lib.bash +++ b/test/lib.bash @@ -693,35 +693,6 @@ function delete_users { rm -fv users.htpasswd } -function add_networkpolicy { - local NAMESPACE=${1:?Pass a namespace as arg[1]} - cat < Date: Tue, 1 Oct 2024 13:16:46 +0200 Subject: [PATCH 04/10] Fix duplicate label on merge --- test/servinge2e/kourier/servicemesh_test.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/servinge2e/kourier/servicemesh_test.go b/test/servinge2e/kourier/servicemesh_test.go index d6a167b524..c452e90a9f 100644 --- a/test/servinge2e/kourier/servicemesh_test.go +++ b/test/servinge2e/kourier/servicemesh_test.go @@ -102,9 +102,6 @@ func TestKsvcWithServiceMeshSidecar(t *testing.T) { // Requests go via gateway -> activator -> pod , by default // Verifies the activator can connect to the pod name: "sidecar-via-activator", - labels: map[string]string{ - servicemesh.IstioInjectKey: "true", - }, annotations: map[string]string{ autoscaling.TargetBurstCapacityKey: "-1", }, @@ -116,9 +113,6 @@ func TestKsvcWithServiceMeshSidecar(t *testing.T) { // Requests go via gateway -> pod ( activator should be skipped if burst capacity is disabled and there is at least 1 replica) // Verifies the gateway can connect to the pod directly name: "sidecar-without-activator", - labels: map[string]string{ - servicemesh.IstioInjectKey: "true", - }, annotations: map[string]string{ autoscaling.TargetBurstCapacityKey: "0", autoscaling.MinScaleAnnotationKey: "1", From 73447310c961f343b77470d483f9eb59f303f39c Mon Sep 17 00:00:00 2001 From: Reto Lehmann Date: Wed, 2 Oct 2024 14:23:27 +0200 Subject: [PATCH 05/10] Use transformer to set istio labels/annotations --- hack/lib/serverless.bash | 39 ++------------ .../pkg/common/istio.go | 52 +++++++++++++++++++ .../pkg/eventing/extension.go | 3 ++ .../pkg/serving/extension.go | 3 ++ 4 files changed, 62 insertions(+), 35 deletions(-) create mode 100644 openshift-knative-operator/pkg/common/istio.go diff --git a/hack/lib/serverless.bash b/hack/lib/serverless.bash index 4a0b99d375..2748d8eb8a 100644 --- a/hack/lib/serverless.bash +++ b/hack/lib/serverless.bash @@ -214,17 +214,6 @@ spec: istio: # point these to our own specific gateways now gateway.knative-serving.knative-ingress-gateway: knative-istio-ingressgateway.knative-serving-ingress.svc.cluster.local local-gateway.knative-serving.knative-local-gateway: knative-local-gateway.knative-serving-ingress.svc.cluster.local - deployments: - - labels: - sidecar.istio.io/inject: "true" - annotations: - sidecar.istio.io/rewriteAppHTTPProbers: "true" - name: activator - - labels: - sidecar.istio.io/inject: "true" - annotations: - sidecar.istio.io/rewriteAppHTTPProbers: "true" - name: autoscaler EOF yq merge --inplace --arrays append "$custom_resource" "$istio_patch" @@ -232,10 +221,6 @@ EOF rm -f "${istio_patch}" } -# If ServiceMesh is enabled: -# - Set ingress.istio.enabled to "true" -# - Set inject and rewriteAppHTTPProbers annotations for activator and autoscaler -# as "test/v1beta1/resources/operator.knative.dev_v1beta1_knativeserving_cr.yaml" has the value "prometheus". function enable_istio_eventing { local custom_resource istio_patch custom_resource=${1:?Pass a custom resource to be patched as arg[1]} @@ -248,29 +233,17 @@ spec: istio: "enabled" delivery-timeout: "enabled" workloads: - - labels: - sidecar.istio.io/inject: "true" - annotations: + - annotations: sidecar.istio.io/logLevel: "debug" - sidecar.istio.io/rewriteAppHTTPProbers: "true" name: pingsource-mt-adapter - - labels: - sidecar.istio.io/inject: "true" - annotations: + - annotations: sidecar.istio.io/logLevel: "debug" - sidecar.istio.io/rewriteAppHTTPProbers: "true" name: mt-broker-ingress - - labels: - sidecar.istio.io/inject: "true" - annotations: + - annotations: sidecar.istio.io/logLevel: "debug" - sidecar.istio.io/rewriteAppHTTPProbers: "true" name: mt-broker-filter - - labels: - sidecar.istio.io/inject: "true" - annotations: + - annotations: sidecar.istio.io/logLevel: "debug" - sidecar.istio.io/rewriteAppHTTPProbers: "true" name: imc-dispatcher EOF @@ -299,10 +272,6 @@ EOF rm -f "${network_patch}" } -# If ServiceMesh is enabled: -# - Set ingress.istio.enabled to "true" -# - Set inject and rewriteAppHTTPProbers annotations for activator and autoscaler -# as "test/v1beta1/resources/operator.knative.dev_v1beta1_knativeserving_cr.yaml" has the value "prometheus". function enable_istio_eventing_kafka { local custom_resource istio_patch custom_resource=${1:?Pass a custom resource to be patched as arg[1]} diff --git a/openshift-knative-operator/pkg/common/istio.go b/openshift-knative-operator/pkg/common/istio.go new file mode 100644 index 0000000000..1ba8762637 --- /dev/null +++ b/openshift-knative-operator/pkg/common/istio.go @@ -0,0 +1,52 @@ +package common + +import ( + "slices" + + mf "github.com/manifestival/manifestival" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/kubernetes/scheme" + "knative.dev/operator/pkg/apis/operator/base" +) + +const ( + istioSidecarInjectionLabel = "sidecar.istio.io/inject" + istioSidecarRewriteProbesAnnotation = "sidecar.istio.io/rewriteAppHTTPProbers" +) + +var deploymentsWithSidecarInjection = []string{ + // Serving + "activator", "autoscaler", + // Eventing + "pingsource-mt-adapter", "mt-broker-ingress", "mt-broker-filter", "imc-dispatcher", +} + +func AddIstioSidecarInjectLabels(kcomp base.KComponent) mf.Transformer { + return func(u *unstructured.Unstructured) error { + if u.GetKind() != "Deployment" { + return nil + } + + deploy := &appsv1.Deployment{} + if err := scheme.Scheme.Convert(u, deploy, nil); err != nil { + return err + } + + if deploy.Spec.Template.ObjectMeta.Labels == nil { + deploy.Spec.Template.ObjectMeta.Labels = map[string]string{} + } + if deploy.Spec.Template.ObjectMeta.Annotations == nil { + deploy.Spec.Template.ObjectMeta.Annotations = map[string]string{} + } + + if slices.Contains(deploymentsWithSidecarInjection, deploy.Name) { + deploy.Spec.Template.ObjectMeta.Labels[istioSidecarInjectionLabel] = "true" + deploy.Spec.Template.ObjectMeta.Annotations[istioSidecarRewriteProbesAnnotation] = "true" + } else { + deploy.Spec.Template.ObjectMeta.Labels[istioSidecarInjectionLabel] = "false" + } + + return scheme.Scheme.Convert(deploy, u, nil) + } +} diff --git a/openshift-knative-operator/pkg/eventing/extension.go b/openshift-knative-operator/pkg/eventing/extension.go index 59cc49bd1e..5f524a8aba 100644 --- a/openshift-knative-operator/pkg/eventing/extension.go +++ b/openshift-knative-operator/pkg/eventing/extension.go @@ -95,6 +95,9 @@ func (e *extension) Transformers(ke base.KComponent) []mf.Transformer { common.ApplyCABundlesTransform(), common.JobsRemoveTTLSecondsAfterFinished(), } + if eventingistio.IsEnabled(ke.GetSpec().GetConfig()) { + tf = append(tf, common.AddIstioSidecarInjectLabels(ke)) + } tf = append(tf, monitoring.GetEventingTransformers(ke)...) return append(tf, common.DeprecatedAPIsTranformers(e.kubeclient.Discovery())...) } diff --git a/openshift-knative-operator/pkg/serving/extension.go b/openshift-knative-operator/pkg/serving/extension.go index 5cabf65aa4..71e9e0dad3 100644 --- a/openshift-knative-operator/pkg/serving/extension.go +++ b/openshift-knative-operator/pkg/serving/extension.go @@ -90,6 +90,9 @@ func (e *extension) Transformers(ks base.KComponent) []mf.Transformer { } tf = append(tf, enableSecretInformerFilteringTransformers(ks)...) tf = append(tf, monitoring.GetServingTransformers(ks)...) + if ks.(*operatorv1beta1.KnativeServing).Spec.Ingress.Istio.Enabled { + tf = append(tf, common.AddIstioSidecarInjectLabels(ks)) + } return append(tf, common.DeprecatedAPIsTranformers(e.kubeclient.Discovery())...) } From 0d54997fa792d4d8a7e343854ca7a39862c8b544 Mon Sep 17 00:00:00 2001 From: Reto Lehmann Date: Wed, 2 Oct 2024 14:26:31 +0200 Subject: [PATCH 06/10] Re-enable eventing and kafka tests --- test/e2e-tests.sh | 18 +++++++++--------- test/upstream-e2e-tests.sh | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/test/e2e-tests.sh b/test/e2e-tests.sh index f6dd2dc657..5dc9d988fb 100755 --- a/test/e2e-tests.sh +++ b/test/e2e-tests.sh @@ -26,21 +26,21 @@ fi run_testselect serverless_operator_e2e_tests -#if [[ $TEST_KNATIVE_KAFKA == true ]]; then -# serverless_operator_kafka_e2e_tests -#fi +if [[ $TEST_KNATIVE_KAFKA == true ]]; then + serverless_operator_kafka_e2e_tests +fi [ -n "$OPENSHIFT_CI" ] && setup_quick_api_deprecation_alerts # Run Knative Serving & Eventing downstream E2E tests. downstream_serving_e2e_tests -#downstream_eventing_e2e_tests -#downstream_eventing_e2e_rekt_tests +downstream_eventing_e2e_tests +downstream_eventing_e2e_rekt_tests downstream_monitoring_e2e_tests -#if [[ $TEST_KNATIVE_KAFKA == true ]]; then -# downstream_knative_kafka_e2e_tests -# downstream_knative_kafka_e2e_rekt_tests -#fi +if [[ $TEST_KNATIVE_KAFKA == true ]]; then + downstream_knative_kafka_e2e_tests + downstream_knative_kafka_e2e_rekt_tests +fi [ -n "$OPENSHIFT_CI" ] && check_serverless_alerts diff --git a/test/upstream-e2e-tests.sh b/test/upstream-e2e-tests.sh index b39f7270d9..c9e5bb4395 100755 --- a/test/upstream-e2e-tests.sh +++ b/test/upstream-e2e-tests.sh @@ -25,9 +25,9 @@ run_testselect # Run upgrade tests if [[ $TEST_KNATIVE_UPGRADE == true ]]; then # Set KafkaChannel as default for upgrade tests. -# if [[ $TEST_KNATIVE_KAFKA == "true" ]]; then -# ensure_kafka_channel_default -# fi + if [[ $TEST_KNATIVE_KAFKA == "true" ]]; then + ensure_kafka_channel_default + fi run_rolling_upgrade_tests fi @@ -36,15 +36,15 @@ if [[ $TEST_KNATIVE_E2E == true ]]; then # TODO: Remove this when upstream tests can use in-cluster config. # See https://github.com/knative/eventing/issues/5996 (the same issue affects Eventing Kafka) ensure_kubeconfig -# if [[ $TEST_KNATIVE_KAFKA_BROKER == true ]]; then -# upstream_knative_eventing_kafka_broker_e2e -# fi + if [[ $TEST_KNATIVE_KAFKA_BROKER == true ]]; then + upstream_knative_eventing_kafka_broker_e2e + fi if [[ $TEST_KNATIVE_SERVING == true ]]; then upstream_knative_serving_e2e_and_conformance_tests fi -# if [[ $TEST_KNATIVE_EVENTING == true ]]; then -# upstream_knative_eventing_e2e -# fi + if [[ $TEST_KNATIVE_EVENTING == true ]]; then + upstream_knative_eventing_e2e + fi fi [ -n "$OPENSHIFT_CI" ] && check_serverless_alerts From a5c08564620d0e1b03983b754acd64bfc76d1357 Mon Sep 17 00:00:00 2001 From: Reto Lehmann Date: Thu, 3 Oct 2024 09:38:04 +0200 Subject: [PATCH 07/10] Add install-tool-sobranch to install-for-mesh-e2e target --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 13598fc975..d0862e2153 100644 --- a/Makefile +++ b/Makefile @@ -154,7 +154,7 @@ test-upstream-e2e-mesh-testonly: install-tool-sobranch MESH=true TEST_KNATIVE_KAFKA=true ./test/e2e-tests.sh MESH=true TEST_KNATIVE_KAFKA=false TEST_KNATIVE_SERVING=true TEST_KNATIVE_EVENTING=true TEST_KNATIVE_KAFKA_BROKER=true TEST_KNATIVE_UPGRADE=false ./test/upstream-e2e-tests.sh -install-for-mesh-e2e: +install-for-mesh-e2e: install-tool-sobranch UNINSTALL_MESH="false" ./hack/mesh.sh TRACING_BACKEND=zipkin TRACING_NAMESPACE=knative-eventing ./hack/tracing.sh UNINSTALL_STRIMZI="false" ./hack/strimzi.sh From 9b4799f120e9cb9fbce35622aa6f995356fd74b5 Mon Sep 17 00:00:00 2001 From: Reto Lehmann Date: Mon, 7 Oct 2024 11:18:59 +0200 Subject: [PATCH 08/10] Undo code changes --- hack/lib/serverless.bash | 31 +++++++++-- .../pkg/common/istio.go | 52 ------------------- .../pkg/eventing/extension.go | 3 -- .../pkg/serving/extension.go | 3 -- 4 files changed, 27 insertions(+), 62 deletions(-) delete mode 100644 openshift-knative-operator/pkg/common/istio.go diff --git a/hack/lib/serverless.bash b/hack/lib/serverless.bash index 2748d8eb8a..e8e413d82f 100644 --- a/hack/lib/serverless.bash +++ b/hack/lib/serverless.bash @@ -214,6 +214,17 @@ spec: istio: # point these to our own specific gateways now gateway.knative-serving.knative-ingress-gateway: knative-istio-ingressgateway.knative-serving-ingress.svc.cluster.local local-gateway.knative-serving.knative-local-gateway: knative-local-gateway.knative-serving-ingress.svc.cluster.local + deployments: + - labels: + sidecar.istio.io/inject: "true" + annotations: + sidecar.istio.io/rewriteAppHTTPProbers: "true" + name: activator + - labels: + sidecar.istio.io/inject: "true" + annotations: + sidecar.istio.io/rewriteAppHTTPProbers: "true" + name: autoscaler EOF yq merge --inplace --arrays append "$custom_resource" "$istio_patch" @@ -233,17 +244,29 @@ spec: istio: "enabled" delivery-timeout: "enabled" workloads: - - annotations: + - labels: + sidecar.istio.io/inject: "true" + annotations: sidecar.istio.io/logLevel: "debug" + sidecar.istio.io/rewriteAppHTTPProbers: "true" name: pingsource-mt-adapter - - annotations: + - labels: + sidecar.istio.io/inject: "true" + annotations: sidecar.istio.io/logLevel: "debug" + sidecar.istio.io/rewriteAppHTTPProbers: "true" name: mt-broker-ingress - - annotations: + - labels: + sidecar.istio.io/inject: "true" + annotations: sidecar.istio.io/logLevel: "debug" + sidecar.istio.io/rewriteAppHTTPProbers: "true" name: mt-broker-filter - - annotations: + - labels: + sidecar.istio.io/inject: "true" + annotations: sidecar.istio.io/logLevel: "debug" + sidecar.istio.io/rewriteAppHTTPProbers: "true" name: imc-dispatcher EOF diff --git a/openshift-knative-operator/pkg/common/istio.go b/openshift-knative-operator/pkg/common/istio.go deleted file mode 100644 index 1ba8762637..0000000000 --- a/openshift-knative-operator/pkg/common/istio.go +++ /dev/null @@ -1,52 +0,0 @@ -package common - -import ( - "slices" - - mf "github.com/manifestival/manifestival" - appsv1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/client-go/kubernetes/scheme" - "knative.dev/operator/pkg/apis/operator/base" -) - -const ( - istioSidecarInjectionLabel = "sidecar.istio.io/inject" - istioSidecarRewriteProbesAnnotation = "sidecar.istio.io/rewriteAppHTTPProbers" -) - -var deploymentsWithSidecarInjection = []string{ - // Serving - "activator", "autoscaler", - // Eventing - "pingsource-mt-adapter", "mt-broker-ingress", "mt-broker-filter", "imc-dispatcher", -} - -func AddIstioSidecarInjectLabels(kcomp base.KComponent) mf.Transformer { - return func(u *unstructured.Unstructured) error { - if u.GetKind() != "Deployment" { - return nil - } - - deploy := &appsv1.Deployment{} - if err := scheme.Scheme.Convert(u, deploy, nil); err != nil { - return err - } - - if deploy.Spec.Template.ObjectMeta.Labels == nil { - deploy.Spec.Template.ObjectMeta.Labels = map[string]string{} - } - if deploy.Spec.Template.ObjectMeta.Annotations == nil { - deploy.Spec.Template.ObjectMeta.Annotations = map[string]string{} - } - - if slices.Contains(deploymentsWithSidecarInjection, deploy.Name) { - deploy.Spec.Template.ObjectMeta.Labels[istioSidecarInjectionLabel] = "true" - deploy.Spec.Template.ObjectMeta.Annotations[istioSidecarRewriteProbesAnnotation] = "true" - } else { - deploy.Spec.Template.ObjectMeta.Labels[istioSidecarInjectionLabel] = "false" - } - - return scheme.Scheme.Convert(deploy, u, nil) - } -} diff --git a/openshift-knative-operator/pkg/eventing/extension.go b/openshift-knative-operator/pkg/eventing/extension.go index 5f524a8aba..59cc49bd1e 100644 --- a/openshift-knative-operator/pkg/eventing/extension.go +++ b/openshift-knative-operator/pkg/eventing/extension.go @@ -95,9 +95,6 @@ func (e *extension) Transformers(ke base.KComponent) []mf.Transformer { common.ApplyCABundlesTransform(), common.JobsRemoveTTLSecondsAfterFinished(), } - if eventingistio.IsEnabled(ke.GetSpec().GetConfig()) { - tf = append(tf, common.AddIstioSidecarInjectLabels(ke)) - } tf = append(tf, monitoring.GetEventingTransformers(ke)...) return append(tf, common.DeprecatedAPIsTranformers(e.kubeclient.Discovery())...) } diff --git a/openshift-knative-operator/pkg/serving/extension.go b/openshift-knative-operator/pkg/serving/extension.go index 71e9e0dad3..5cabf65aa4 100644 --- a/openshift-knative-operator/pkg/serving/extension.go +++ b/openshift-knative-operator/pkg/serving/extension.go @@ -90,9 +90,6 @@ func (e *extension) Transformers(ks base.KComponent) []mf.Transformer { } tf = append(tf, enableSecretInformerFilteringTransformers(ks)...) tf = append(tf, monitoring.GetServingTransformers(ks)...) - if ks.(*operatorv1beta1.KnativeServing).Spec.Ingress.Istio.Enabled { - tf = append(tf, common.AddIstioSidecarInjectLabels(ks)) - } return append(tf, common.DeprecatedAPIsTranformers(e.kubeclient.Discovery())...) } From b57c7a11394637e2371a53a5686e65fbf3d191a5 Mon Sep 17 00:00:00 2001 From: Reto Lehmann Date: Mon, 7 Oct 2024 11:41:21 +0200 Subject: [PATCH 09/10] Use istio-injection labels with false to deactivate injection --- hack/lib/serverless.bash | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/hack/lib/serverless.bash b/hack/lib/serverless.bash index e8e413d82f..1e98e8b10c 100644 --- a/hack/lib/serverless.bash +++ b/hack/lib/serverless.bash @@ -225,6 +225,21 @@ spec: annotations: sidecar.istio.io/rewriteAppHTTPProbers: "true" name: autoscaler + - labels: + sidecar.istio.io/inject: "false" + name: autoscaler-hpa + - labels: + sidecar.istio.io/inject: "false" + name: controller + - labels: + sidecar.istio.io/inject: "false" + name: net-istio-controller + - labels: + sidecar.istio.io/inject: "false" + name: net-istio-webhook + - labels: + sidecar.istio.io/inject: "false" + name: webhook EOF yq merge --inplace --arrays append "$custom_resource" "$istio_patch" @@ -268,6 +283,21 @@ spec: sidecar.istio.io/logLevel: "debug" sidecar.istio.io/rewriteAppHTTPProbers: "true" name: imc-dispatcher + - labels: + sidecar.istio.io/inject: "false" + name: eventing-controller + - labels: + sidecar.istio.io/inject: "false" + name: eventing-istio-controller + - labels: + sidecar.istio.io/inject: "false" + name: eventing-webhook + - labels: + sidecar.istio.io/inject: "false" + name: imc-controller + - labels: + sidecar.istio.io/inject: "false" + name: job-sink EOF yq merge --inplace --arrays append "$custom_resource" "$istio_patch" @@ -345,6 +375,9 @@ spec: sidecar.istio.io/logLevel: "debug" sidecar.istio.io/rewriteAppHTTPProbers: "true" name: kafka-controller + - labels: + sidecar.istio.io/inject: "false" + name: kafka-webhook-eventing EOF yq merge --inplace --arrays append "$custom_resource" "$istio_patch" From 998b2bb8632c79417b360d31571ade442298df5c Mon Sep 17 00:00:00 2001 From: Reto Lehmann Date: Mon, 7 Oct 2024 15:28:58 +0200 Subject: [PATCH 10/10] Use istio-injection labels with false to deactivate injection --- hack/lib/serverless.bash | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hack/lib/serverless.bash b/hack/lib/serverless.bash index 1e98e8b10c..4295d3ada7 100644 --- a/hack/lib/serverless.bash +++ b/hack/lib/serverless.bash @@ -298,6 +298,9 @@ spec: - labels: sidecar.istio.io/inject: "false" name: job-sink + - labels: + sidecar.istio.io/inject: "false" + name: mt-broker-controller EOF yq merge --inplace --arrays append "$custom_resource" "$istio_patch"