[TEST-ONLY] Testing Integration with OSSM 3.x

Makefile

@@ -154,7 +154,7 @@ test-upstream-e2e-mesh-testonly: install-tool-sobranch
 	MESH=true TEST_KNATIVE_KAFKA=true ./test/e2e-tests.sh
 	MESH=true TEST_KNATIVE_KAFKA=false TEST_KNATIVE_SERVING=true TEST_KNATIVE_EVENTING=true TEST_KNATIVE_KAFKA_BROKER=true TEST_KNATIVE_UPGRADE=false ./test/upstream-e2e-tests.sh
 
-install-for-mesh-e2e:
+install-for-mesh-e2e: install-tool-sobranch
 	UNINSTALL_MESH="false" ./hack/mesh.sh
 	TRACING_BACKEND=zipkin TRACING_NAMESPACE=knative-eventing ./hack/tracing.sh
 	UNINSTALL_STRIMZI="false" ./hack/strimzi.sh

hack/lib/certmanager.bash

@@ -31,9 +31,6 @@ function deploy_certmanager_operator {
   oc apply -n "${deployment_namespace}" -f "${certmanager_resources_dir}"/serving-ca-certificate.yaml || return $?
 
   sync_trust_bundle "knative-selfsigned-ca" "knative-serving" "knative-serving-ingress" || return $?
-  if [[ $MESH == "true" ]]; then
-    sync_trust_bundle "knative-selfsigned-ca" "istio-system" || return $?
-  fi
 
   # eventing resources
   oc apply -f "${certmanager_resources_dir}"/selfsigned-issuer.yaml || return $?
@@ -67,7 +64,7 @@ function sync_trust_bundle {
 
    for ns in "${namespaces[@]}"; do
      echo "Syncing trust-bundle for namespace: ${ns}"
-     oc create namespace "${ns}" --dry-run=client -o yaml | oc apply -f -
+     oc get namespace "${ns}" || oc create namespace "${ns}"
      oc label namespace "${ns}" knative.openshift.io/part-of="openshift-serverless" --overwrite
      oc create configmap -n "${ns}" knative-ca-bundle --from-file=tls.crt --from-file=ca.crt \
         --dry-run=client -o yaml | kubectl apply -n "${ns}" -f - || return $?

hack/lib/mesh.bash

@@ -22,65 +22,63 @@ function deploy_servicemesh_operators {
   fi
 
   logger.info "Waiting until service mesh operators are available"
-  timeout 600 "[[ \$(oc get deploy -n openshift-operators istio-operator --no-headers | wc -l) != 1 ]]" || return 1
-  oc wait --for=condition=Available deployment istio-operator --timeout=300s -n openshift-operators || return $?
+  timeout 600 "[[ \$(oc get deploy -n openshift-operators sail-operator --no-headers | wc -l) != 1 ]]" || return 1
+  oc wait --for=condition=Available deployment sail-operator --timeout=300s -n openshift-operators || return $?
 }
 
 function undeploy_servicemesh_operators {
   logger.info "Deleting service mesh subscriptions"
-  oc delete subscriptions.operators.coreos.com -n openshift-operators servicemeshoperator kiali-ossm jaeger-product --ignore-not-found
+  oc delete subscriptions.operators.coreos.com -n openshift-operators sailoperator --ignore-not-found
   logger.info 'Deleting ClusterServiceVersion'
   for csv in $(set +o pipefail && oc get csv -n openshift-operators --no-headers 2>/dev/null \
-      | grep 'servicemeshoperator\|jaeger\|kiali' | cut -f1 -d' '); do
+      | grep 'sailoperator' | cut -f1 -d' '); do
     oc delete csv -n openshift-operators "${csv}"
   done
 
   logger.info 'Ensure no operators present'
-  timeout 600 "[[ \$(oc get deployments -n openshift-operators -oname | grep -c 'servicemeshoperator\|jaeger\|kiali') != 0 ]]"
-
-  logger.info "Deleting service mesh istio nodes"
-  oc delete --ignore-not-found=true daemonset.apps/istio-node -n openshift-operators
-  oc delete --ignore-not-found=true service/maistra-admission-controller -n openshift-operators
+  timeout 600 "[[ \$(oc get deployments -n openshift-operators -oname | grep -c 'sail-operator') != 0 ]]"
 
   logger.info "Deleting service mesh webhooks and rbac resources"
-  oc delete --ignore-not-found=true validatingwebhookconfiguration openshift-operators.servicemesh-resources.maistra.io
-  oc delete --ignore-not-found=true mutatingwebhookconfigurations openshift-operators.servicemesh-resources.maistra.io
-  oc delete --ignore-not-found=true clusterrole istio-admin istio-cni istio-edit istio-view
-  oc delete --ignore-not-found=true clusterrolebinding istio-cn
+  oc delete --ignore-not-found=true clusterrole istio-admin istio-edit istio-view
 
   logger.info 'Ensure not CRDs left'
-  if [[ ! $(oc get crd -oname | grep -c 'maistra.io') -eq 0 ]]; then
-    oc get crd -oname | grep 'maistra.io' | xargs oc delete --timeout=60s
-  fi
   if [[ ! $(oc get crd -oname | grep -c 'istio') -eq 0 ]]; then
     oc get crd -oname | grep 'istio' | xargs oc delete --timeout=60s
   fi
   logger.success "Service mesh has been uninstalled"
 }
 
 function deploy_servicemeshcontrolplane {
-  logger.info "Installing ServiceMeshControlPlane in namespace istio-system"
+  logger.info "Installing istiod in namespace istio-system"
 
   oc get ns istio-system || oc create namespace istio-system
+  oc get ns istio-cni || oc create namespace istio-cni
 
-  # Make sure servicemeshcontrolplanes.maistra.io is available.
-  timeout 120 "[[ \$(oc get crd servicemeshcontrolplanes.maistra.io --no-headers | wc -l) != 1 ]]" || return 1
-  oc wait --for=condition=Established crd servicemeshcontrolplanes.maistra.io
+  # Make sure istios.operator.istio.io is available.
+  timeout 120 "[[ \$(oc get crd istios.operator.istio.io --no-headers | wc -l) != 1 ]]" || return 1
+  oc wait --for=condition=Established crd istios.operator.istio.io
 
-  # creating smcp often fails due to webhook error
-  timeout 120 "[[ \$(oc apply -f ${resources_dir}/smcp.yaml | oc get smcp -n istio-system basic --no-headers | wc -l) != 1 ]]" || return 1
+  timeout 120 "[[ \$(oc apply -f ${resources_dir}/istio.yaml | oc get istios -n istio-system default --no-headers | wc -l) != 1 ]]" || return 1
+  timeout 120 "[[ \$(oc apply -f ${resources_dir}/istio-cni.yaml | oc get istiocnis -n default default --no-headers | wc -l) != 1 ]]" || return 1
 
-  if [[ $(oc get infrastructure cluster -ojsonpath='{.status.platformStatus.aws.resourceTags[?(@.key=="red-hat-clustertype")].value}') = rosa ]]; then
-    logger.info "ThirdParty tokens required when using ROSA cluster"
-    enable_smcp_third_party_token
-  fi
+# TODO: CHECK ME for OSSM3
+#  if [[ $(oc get infrastructure cluster -ojsonpath='{.status.platformStatus.aws.resourceTags[?(@.key=="red-hat-clustertype")].value}') = rosa ]]; then
+#    logger.info "ThirdParty tokens required when using ROSA cluster"
+#    enable_smcp_third_party_token
+#  fi
+
+  oc wait --timeout=180s --for=condition=Ready istios -n istio-system default || oc get istios -n istio-system default -o yaml
+  oc wait --timeout=180s --for=condition=Ready istiocnis -n default default || oc get istiocnis -n default default -o yaml
 
-  oc wait --timeout=180s --for=condition=Ready smcp -n istio-system basic || oc get smcp -n istio-system basic -o yaml
+  # make sure istiod + cni pods are up before continuing
+  oc wait deploy --all --timeout=600s --for=condition=Available -n istio-system
+  oc rollout status daemonset -n istio-cni --timeout 600s
 }
 
 function undeploy_servicemeshcontrolplane {
-  logger.info "Deleting ServiceMeshControlPlane"
-  oc delete smcp -n istio-system basic --ignore-not-found || return $?
+  logger.info "Deleting istiod"
+  oc delete istios default -n istio-system --ignore-not-found || return $?
+  oc delete istiocnis default --ignore-not-found || return $?
 }
 
 function deploy_gateways {
@@ -109,7 +107,9 @@ function deploy_gateways {
       -in "${out_dir}"/wildcard.csr \
       -out "${out_dir}"/wildcard.crt
 
-  oc create -n istio-system secret tls wildcard-certs \
+  oc apply -f "${resources_dir}"/namespace.yaml || return $?
+
+  oc create -n knative-serving-ingress secret tls wildcard-certs \
       --key="${out_dir}"/wildcard.key \
       --cert="${out_dir}"/wildcard.crt --dry-run=client -o yaml | oc apply -f - 
 
@@ -119,32 +119,31 @@ function deploy_gateways {
       --key="${out_dir}"/wildcard.key \
       --cert="${out_dir}"/wildcard.crt --dry-run=client -o yaml | oc apply -f -
 
-  oc apply -f "${resources_dir}"/namespace.yaml || return $?
-  oc apply -f "${resources_dir}"/smmr.yaml || return $?
+  oc apply -f "${resources_dir}"/gateway-deploy.yaml || return $?
   oc apply -f "${resources_dir}"/gateway.yaml || return $?
   oc apply -f "${resources_dir}"/authorization-policies/setup || return $?
   oc apply -f "${resources_dir}"/authorization-policies/helm || return $?
   oc apply -f "${resources_dir}"/destination-rules.yaml || return $?
+  oc apply -f "${resources_dir}"/peer-authentication-mesh-mtls.yaml || return $?
 
   oc apply -n "${EVENTING_NAMESPACE}" -f "${resources_dir}"/kafka-service-entry.yaml || return $?
   for ns in serverless-tests eventing-e2e0 eventing-e2e1 eventing-e2e2 eventing-e2e3 eventing-e2e4; do
     oc apply -n "$ns" -f "${resources_dir}"/kafka-service-entry.yaml || return $?
   done
-  oc apply -n "serverless-tests" -f "${resources_dir}"/network-policy-monitoring.yaml || return $?
 }
 
 function undeploy_gateways {
-  oc delete -n serverless-tests -f "${resources_dir}"/network-policy-monitoring.yaml --ignore-not-found || return $?
   for ns in serverless-tests eventing-e2e0 eventing-e2e1 eventing-e2e2 eventing-e2e3 eventing-e2e4; do
     oc delete -n "$ns" -f "${resources_dir}"/kafka-service-entry.yaml --ignore-not-found || return $?
   done
+  oc delete -f "${resources_dir}"/peer-authentication-mesh-mtls.yaml --ignore-not-found || return $?
   oc delete -f "${resources_dir}"/destination-rules.yaml --ignore-not-found || return $?
   oc delete -f "${resources_dir}"/authorization-policies/helm --ignore-not-found || return $?
   oc delete -f "${resources_dir}"/authorization-policies/setup --ignore-not-found || return $?
   oc delete -f "${resources_dir}"/gateway.yaml --ignore-not-found || return $?
-  oc delete -f "${resources_dir}"/smmr.yaml --ignore-not-found || return $?
+  oc delete -f "${resources_dir}"/gateway-deploy.yaml --ignore-not-found || return $?
   oc delete -n cert-manager secret ca-key-pair  --ignore-not-found || return $?
-  oc delete -n istio-system secret wildcard-certs --ignore-not-found || return $?
+  oc delete -n knative-serving-ingress secret wildcard-certs --ignore-not-found || return $?
 }
 
 function enable_smcp_third_party_token {

hack/lib/mesh_resources/authorization-policies/setup/serving-allow-traffic-to-activator.yaml

@@ -11,4 +11,4 @@ spec:
   rules:
     - from:
         - source:
-            namespaces: [ "knative-serving", "istio-system" ]
+            namespaces: [ "knative-serving", "knative-serving-ingress" ]

hack/lib/mesh_resources/destination-rules.yaml

@@ -10,3 +10,16 @@ spec:
     tls:
       mode: DISABLE
 ---
+# This DR forces mTLS to be used for DomainMappings which are routed from
+# knative-gateway to knative-local-gateway in knative-serving-ingress
+apiVersion: networking.istio.io/v1beta1
+kind: DestinationRule
+metadata:
+  name: local-gateway-force-mtls
+  namespace: knative-serving-ingress
+spec:
+  host: 'knative-local-gateway.knative-serving-ingress.svc.cluster.local'
+  trafficPolicy:
+    tls:
+      mode: ISTIO_MUTUAL
+---

hack/lib/mesh_resources/gateway-deploy.yaml

@@ -0,0 +1,50 @@
+# based on OSSM 3.0 samples: https://raw.githubusercontent.com/maistra/istio-operator/maistra-3.0/chart/samples/ingress-gateway.yaml
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: knative-istio-ingressgateway
+  namespace: knative-serving-ingress
+spec:
+  selector:
+    matchLabels:
+      knative: ingressgateway
+  template:
+    metadata:
+      annotations:
+        # Select the gateway injection template (rather than the default sidecar template)
+        inject.istio.io/templates: gateway
+      labels:
+        # Set a unique label for the gateway. This is required to ensure Gateways can select this workload
+        knative: ingressgateway
+        sidecar.istio.io/inject: "true"
+    spec:
+      containers:
+        - name: istio-proxy
+          image: auto # The image will automatically update each time the pod starts.
+
+---
+# Set up roles to allow reading credentials for TLS
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: istio-ingressgateway-sds
+  namespace: knative-serving-ingress
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "watch", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: istio-ingressgateway-sds
+  namespace: knative-serving-ingress
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: istio-ingressgateway-sds
+subjects:
+  - kind: ServiceAccount
+    name: default
+---

hack/lib/mesh_resources/gateway.yaml

@@ -1,55 +1,82 @@
-apiVersion: networking.istio.io/v1alpha3
-kind: Gateway
+###########################################################
+# cluster external
+###########################################################
+apiVersion: v1
+kind: Service
 metadata:
-  name: knative-ingress-gateway
-  namespace: knative-serving
+  name: knative-istio-ingressgateway
+  namespace: knative-serving-ingress
 spec:
+  type: ClusterIP
   selector:
     knative: ingressgateway
-  servers:
-    - port:
-        number: 443
-        name: https
-        protocol: HTTPS
-      hosts:
-        - "*"
-      tls:
-        mode: SIMPLE
-        credentialName: wildcard-certs
+  ports:
+    - name: http2
+      port: 80
+      targetPort: 8080
+    - name: https
+      port: 443
+      targetPort: 8443
 ---
-apiVersion: networking.istio.io/v1alpha3
+apiVersion: networking.istio.io/v1beta1
 kind: Gateway
 metadata:
-  name: knative-local-gateway
+  name: knative-ingress-gateway
   namespace: knative-serving
 spec:
   selector:
     knative: ingressgateway
   servers:
-    - port:
-        number: 8081
+    - hosts:
+        - '*'
+      port:
         name: https
+        number: 443
         protocol: HTTPS
       tls:
-        mode: ISTIO_MUTUAL
-      hosts:
-        - "*"
+        credentialName: wildcard-certs
+        mode: SIMPLE
 ---
+###########################################################
+# cluster local
+###########################################################
 apiVersion: v1
 kind: Service
 metadata:
-  name: knative-local-gateway
-  namespace: istio-system
   labels:
     experimental.istio.io/disable-gateway-port-translation: "true"
+  name: knative-local-gateway
+  namespace: knative-serving-ingress
 spec:
-  type: ClusterIP
-  selector:
-    istio: ingressgateway
   ports:
     - name: http2
       port: 80
+      protocol: TCP
       targetPort: 8081
     - name: https
       port: 443
+      protocol: TCP
       targetPort: 8444
+  selector:
+    knative: ingressgateway
+  type: ClusterIP
+#########################
+---
+apiVersion: networking.istio.io/v1alpha3
+kind: Gateway
+metadata:
+  name: knative-local-gateway
+  namespace: knative-serving
+spec:
+  selector:
+    knative: ingressgateway
+  servers:
+    - hosts:
+        - "*"
+      port:
+        number: 8081
+        name: https
+        protocol: HTTPS
+      tls:
+        mode: ISTIO_MUTUAL
+---

hack/lib/mesh_resources/istio-cni.yaml

@@ -0,0 +1,7 @@
+kind: IstioCNI
+apiVersion: operator.istio.io/v1alpha1
+metadata:
+  name: default
+spec:
+  namespace: istio-cni
+  version: v1.21.0

hack/lib/mesh_resources/istio.yaml

@@ -0,0 +1,17 @@
+apiVersion: operator.istio.io/v1alpha1
+kind: Istio
+metadata:
+  name: default # this is propagated to the revision and "istio-injection: enabled" only works with this name
+spec:
+  values:
+    global:
+      proxy:
+        excludeInboundPorts: "8444,8022" # metrics, serving: wait-for-drain k8s pre-stop hook
+    meshConfig:
+      defaultConfig:
+        terminationDrainDuration: 35s # needed to make QP stop hook work
+  updateStrategy:
+    inactiveRevisionDeletionGracePeriodSeconds: 30
+    type: InPlace
+  namespace: istio-system
+  version: v1.21.0