Skip to content

Commit

Permalink
ensure cdi transient pods handle mesh side proxies on shutdown
Browse files Browse the repository at this point in the history
  • Loading branch information
bc185174 committed Apr 3, 2024
1 parent c7a457f commit 4d49377
Show file tree
Hide file tree
Showing 21 changed files with 308 additions and 24 deletions.
34 changes: 12 additions & 22 deletions BUILD.bazel
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
load("@io_bazel_rules_docker//container:container.bzl", "container_image")
load("@bazel_gazelle//:def.bzl", "gazelle")
load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair")
Expand Down Expand Up @@ -34,13 +33,6 @@ gazelle(name = "gazelle")

bazeldnf(name = "bazeldnf")

go_library(
name = "go_default_library",
srcs = ["doc.go"],
importpath = "kubevirt.io/containerized-data-importer",
visibility = ["//visibility:public"],
)

load(
"@io_bazel_rules_docker//container:container.bzl",
"container_bundle",
Expand Down Expand Up @@ -90,14 +82,12 @@ container_bundle(

load("@io_bazel_rules_docker//contrib:push-all.bzl", "docker_push")



alias(
name = "test-container-images",
actual = select({
"@io_bazel_rules_go//go/platform:linux_arm64":":test-container-images-aarch64",
"//conditions:default": ":test-container-images-amd64",
})
"@io_bazel_rules_go//go/platform:linux_arm64": ":test-container-images-aarch64",
"//conditions:default": ":test-container-images-amd64",
}),
)

docker_push(
Expand Down Expand Up @@ -171,17 +161,17 @@ container_push(
filegroup(
name = "test-images",
srcs = [
"//tests:images/tinyCore.iso",
"//tests:images/tinyCore.vdi",
"//tests:images/archive.tar",
"//tests:images/cirros-qcow2.img",
"//tests:images/cirros.raw",
"//tests:images/cirros-large-virtual-size.raw.xz",
"//tests:images/cirros-large-virtual-size.qcow2",
"//tests:images/cirros-large-physical-size.raw.xz",
"//tests:images/cirros-large-physical-size.qcow2",
"//tests:images/cirros-snapshot1.qcow2",
"//tests:images/cirros-snapshot2.qcow2",
"//tests:images/cirros-large-physical-size.qcow2",
"//tests:images/cirros-large-physical-size.raw.xz",
"//tests:images/cirros-large-virtual-size.qcow2",
"//tests:images/cirros-large-virtual-size.raw.xz",
"//tests:images/cirros-qcow2.img",
"//tests:images/cirros-snapshot1.qcow2",
"//tests:images/cirros-snapshot2.qcow2",
"//tests:images/tinyCore.iso",
"//tests:images/tinyCore.vdi",
],
visibility = ["//visibility:public"],
)
Expand Down
4 changes: 4 additions & 0 deletions pkg/controller/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ go_library(
"//pkg/controller/common:go_default_library",
"//pkg/controller/datavolume:go_default_library",
"//pkg/feature-gates:go_default_library",
"//pkg/mesh/istio:go_default_library",
"//pkg/mesh/linkerd:go_default_library",
"//pkg/monitoring/metrics/cdi-controller:go_default_library",
"//pkg/operator:go_default_library",
"//pkg/storagecapabilities:go_default_library",
Expand Down Expand Up @@ -88,6 +90,8 @@ go_test(
"//pkg/controller/common:go_default_library",
"//pkg/controller/datavolume:go_default_library",
"//pkg/feature-gates:go_default_library",
"//pkg/mesh/istio:go_default_library",
"//pkg/mesh/linkerd:go_default_library",
"//pkg/monitoring/metrics/cdi-controller:go_default_library",
"//pkg/operator:go_default_library",
"//pkg/storagecapabilities:go_default_library",
Expand Down
13 changes: 13 additions & 0 deletions pkg/controller/clone-controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/go-logr/logr"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
Expand All @@ -33,6 +34,8 @@ import (
cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1"
"kubevirt.io/containerized-data-importer/pkg/common"
cc "kubevirt.io/containerized-data-importer/pkg/controller/common"
istio "kubevirt.io/containerized-data-importer/pkg/mesh/istio"
l5d "kubevirt.io/containerized-data-importer/pkg/mesh/linkerd"
"kubevirt.io/containerized-data-importer/pkg/util"
"kubevirt.io/containerized-data-importer/pkg/util/cert/fetcher"
"kubevirt.io/containerized-data-importer/pkg/util/cert/generator"
Expand Down Expand Up @@ -719,6 +722,16 @@ func MakeCloneSourcePodSpec(sourceVolumeMode corev1.PersistentVolumeMode, image,
}
}

if l5d.IsMeshed(targetPvc.GetAnnotations()) {
pod.Spec.Containers[0].Lifecycle = &v1.Lifecycle{
PreStop: l5d.PreStopHook(),
}
} else if istio.IsMeshed(targetPvc.GetAnnotations()) {
pod.Spec.Containers[0].Lifecycle = &v1.Lifecycle{
PreStop: istio.PreStopHook(),
}
}

pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, addVars...)
cc.SetPvcAllowedAnnotations(pod, targetPvc)
cc.SetRestrictedSecurityContext(&pod.Spec)
Expand Down
37 changes: 37 additions & 0 deletions pkg/controller/clone-controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package controller
import (
"context"
"fmt"
"maps"
"reflect"
"strings"
"time"
Expand All @@ -26,6 +27,7 @@ import (
. "github.com/onsi/gomega"

corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
Expand All @@ -39,6 +41,8 @@ import (
cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1"
"kubevirt.io/containerized-data-importer/pkg/common"
cc "kubevirt.io/containerized-data-importer/pkg/controller/common"
istio "kubevirt.io/containerized-data-importer/pkg/mesh/istio"
"kubevirt.io/containerized-data-importer/pkg/mesh/linkerd"
"kubevirt.io/containerized-data-importer/pkg/token"
"kubevirt.io/containerized-data-importer/pkg/util/cert/fetcher"
)
Expand Down Expand Up @@ -297,6 +301,39 @@ var _ = Describe("Clone controller reconcile loop", func() {
}),
)

DescribeTable("should create clone pod with service mesh lifecycle hook", func(annotations map[string]string, expectedHook *v1.Lifecycle) {
defaultAnnotations := map[string]string{
cc.AnnCloneRequest: "default/source",
cc.AnnPodReady: "true",
cc.AnnCloneToken: "foobaz",
AnnUploadClientName: "uploadclient",
cc.AnnCloneSourcePod: "default-testPvc1-source-pod",
}
maps.Copy(defaultAnnotations, annotations)
testPvc := cc.CreatePvc("testPvc1", "default", defaultAnnotations, nil)
sourcePvc := cc.CreatePvc("source", "default", map[string]string{}, nil)
pod := podUsingBlockPVC(sourcePvc, false)
pod.Name = "other-target-pvc-uid" + common.ClonerSourcePodNameSuffix
pod.Labels = map[string]string{
common.CDIComponentLabel: common.ClonerSourcePodName,
}
reconciler = createCloneReconciler(testPvc, sourcePvc, pod)
result, err := reconciler.Reconcile(context.TODO(), reconcile.Request{NamespacedName: types.NamespacedName{Name: "testPvc1", Namespace: "default"}})
Expect(err).ToNot(HaveOccurred())
Expect(result.RequeueAfter).To(BeZero())
By("Verifying source pod gets created")
sourcePod, err := reconciler.findCloneSourcePod(testPvc)
Expect(err).ToNot(HaveOccurred())
Expect(sourcePod).ToNot(BeNil())
Expect(sourcePod.Name).ToNot(Equal(pod.Name))
By("Verifying the pod lifecycle hook is set")
Expect(sourcePod.Spec.Containers[0].Lifecycle).To(Equal(expectedHook))
reconciler = nil
},
Entry("should create pod with linkerd mesh lifecycle hook", map[string]string{cc.AnnPodSidecarInjectionLinkerd: "enabled"}, &v1.Lifecycle{PreStop: linkerd.PreStopHook()}),
Entry("should create pod with istio mesh lifecycle hook", map[string]string{cc.AnnPodSidecarInjectionIstio: "true"}, &v1.Lifecycle{PreStop: istio.PreStopHook()}),
)

It("Should error with missing upload client name annotation if none provided", func() {
testPvc := cc.CreatePvc("testPvc1", "default", map[string]string{
cc.AnnCloneRequest: "default/source", cc.AnnPodReady: "true", cc.AnnCloneToken: "foobaz", cc.AnnCloneSourcePod: "default-testPvc1-source-pod"}, nil)
Expand Down
4 changes: 4 additions & 0 deletions pkg/controller/clone/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ go_library(
deps = [
"//pkg/common:go_default_library",
"//pkg/controller/common:go_default_library",
"//pkg/mesh/istio:go_default_library",
"//pkg/mesh/linkerd:go_default_library",
"//pkg/util:go_default_library",
"//staging/src/kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1:go_default_library",
"//vendor/github.com/go-logr/logr:go_default_library",
Expand Down Expand Up @@ -55,6 +57,8 @@ go_test(
embed = [":go_default_library"],
deps = [
"//pkg/controller/common:go_default_library",
"//pkg/mesh/istio:go_default_library",
"//pkg/mesh/linkerd:go_default_library",
"//staging/src/kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1:go_default_library",
"//vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1:go_default_library",
"//vendor/github.com/onsi/ginkgo/v2:go_default_library",
Expand Down
13 changes: 13 additions & 0 deletions pkg/controller/clone/prep-claim.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (

"github.com/go-logr/logr"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/record"
"kubevirt.io/containerized-data-importer/pkg/common"
Expand All @@ -14,6 +15,8 @@ import (
"sigs.k8s.io/controller-runtime/pkg/reconcile"

cc "kubevirt.io/containerized-data-importer/pkg/controller/common"
istio "kubevirt.io/containerized-data-importer/pkg/mesh/istio"
l5d "kubevirt.io/containerized-data-importer/pkg/mesh/linkerd"
)

// PrepClaimPhaseName is the name of the prep claim phase
Expand Down Expand Up @@ -206,6 +209,16 @@ func (p *PrepClaimPhase) createPod(ctx context.Context, name string, pvc *corev1
AddOwnershipLabel(p.OwnershipLabel, pod, p.Owner)
}

if l5d.IsMeshed(pvc.GetAnnotations()) {
pod.Spec.Containers[0].Lifecycle = &v1.Lifecycle{
PreStop: l5d.PreStopHook(),
}
} else if istio.IsMeshed(pvc.GetAnnotations()) {
pod.Spec.Containers[0].Lifecycle = &v1.Lifecycle{
PreStop: istio.PreStopHook(),
}
}

cc.SetRestrictedSecurityContext(&pod.Spec)

if err := p.Client.Create(ctx, pod); err != nil {
Expand Down
26 changes: 26 additions & 0 deletions pkg/controller/clone/prep-claim_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,13 @@ package clone

import (
"context"
"maps"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"

corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -35,6 +37,8 @@ import (

cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1"
cc "kubevirt.io/containerized-data-importer/pkg/controller/common"
istio "kubevirt.io/containerized-data-importer/pkg/mesh/istio"
"kubevirt.io/containerized-data-importer/pkg/mesh/linkerd"
)

var _ = Describe("PrepClaimPhase test", func() {
Expand Down Expand Up @@ -263,6 +267,28 @@ var _ = Describe("PrepClaimPhase test", func() {
Expect(pod.Spec.NodeName).To(Equal(""))
})

DescribeTable("should create prep claim pod with service mesh lifecycle hook", func(annotations map[string]string, expectedHook *v1.Lifecycle) {
claim := getClaim()
cc.AddAnnotation(claim, cc.AnnSelectedNode, "node1")
maps.Copy(claim.Annotations, annotations)
claim.Spec.Resources.Requests[corev1.ResourceStorage] = defaultRequestSize

p := createPrepClaimPhase(claim)

result, err := p.Reconcile(context.Background())
Expect(err).ToNot(HaveOccurred())
Expect(result).ToNot(BeNil())
Expect(result.Requeue).To(BeFalse())
Expect(result.RequeueAfter).To(BeZero())

pod := getCreatedPod(p)
By("Verifying the pod lifecycle hook is set")
Expect(pod.Spec.Containers[0].Lifecycle).To(Equal(expectedHook))
},
Entry("should create pod with linkerd mesh lifecycle hook", map[string]string{cc.AnnPodSidecarInjectionLinkerd: "enabled"}, &v1.Lifecycle{PreStop: linkerd.PreStopHook()}),
Entry("should create pod with istio mesh lifecycle hook", map[string]string{cc.AnnPodSidecarInjectionIstio: "true"}, &v1.Lifecycle{PreStop: istio.PreStopHook()}),
)

Context("with prep pod created", func() {
getPod := func() *corev1.Pod {
return &corev1.Pod{
Expand Down
4 changes: 4 additions & 0 deletions pkg/controller/datavolume/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ go_library(
"//pkg/controller/common:go_default_library",
"//pkg/controller/populators:go_default_library",
"//pkg/feature-gates:go_default_library",
"//pkg/mesh/istio:go_default_library",
"//pkg/mesh/linkerd:go_default_library",
"//pkg/monitoring/metrics/cdi-controller:go_default_library",
"//pkg/token:go_default_library",
"//pkg/util:go_default_library",
Expand Down Expand Up @@ -76,6 +78,8 @@ go_test(
"//pkg/controller/common:go_default_library",
"//pkg/controller/populators:go_default_library",
"//pkg/feature-gates:go_default_library",
"//pkg/mesh/istio:go_default_library",
"//pkg/mesh/linkerd:go_default_library",
"//pkg/token:go_default_library",
"//staging/src/kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1:go_default_library",
"//vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1:go_default_library",
Expand Down
12 changes: 12 additions & 0 deletions pkg/controller/datavolume/pvc-clone-controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/go-logr/logr"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -43,6 +44,8 @@ import (
"kubevirt.io/containerized-data-importer/pkg/common"
cc "kubevirt.io/containerized-data-importer/pkg/controller/common"
featuregates "kubevirt.io/containerized-data-importer/pkg/feature-gates"
istio "kubevirt.io/containerized-data-importer/pkg/mesh/istio"
l5d "kubevirt.io/containerized-data-importer/pkg/mesh/linkerd"
)

const (
Expand Down Expand Up @@ -642,6 +645,15 @@ func (r *PvcCloneReconciler) makeSizeDetectionPodSpec(
if err != nil {
return nil
}
if l5d.IsMeshed(sourcePvc.GetAnnotations()) {
container.Lifecycle = &v1.Lifecycle{
PreStop: l5d.PreStopHook(),
}
} else if istio.IsMeshed(sourcePvc.GetAnnotations()) {
container.Lifecycle = &v1.Lifecycle{
PreStop: istio.PreStopHook(),
}
}

// Assemble the pod
pod := &corev1.Pod{
Expand Down
23 changes: 23 additions & 0 deletions pkg/controller/datavolume/pvc-clone-controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package datavolume

import (
"context"
"maps"
"reflect"
"strings"
"time"
Expand All @@ -27,6 +28,7 @@ import (

snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
Expand All @@ -45,8 +47,11 @@ import (
"kubevirt.io/containerized-data-importer/pkg/common"
"kubevirt.io/containerized-data-importer/pkg/controller/clone"
. "kubevirt.io/containerized-data-importer/pkg/controller/common"
cc "kubevirt.io/containerized-data-importer/pkg/controller/common"
"kubevirt.io/containerized-data-importer/pkg/controller/populators"
featuregates "kubevirt.io/containerized-data-importer/pkg/feature-gates"
istio "kubevirt.io/containerized-data-importer/pkg/mesh/istio"
"kubevirt.io/containerized-data-importer/pkg/mesh/linkerd"
"kubevirt.io/containerized-data-importer/pkg/token"
)

Expand Down Expand Up @@ -855,6 +860,24 @@ var _ = Describe("All DataVolume Tests", func() {
},
Entry("hostAssited with empty size and 'Block' volume mode", cdiv1.CloneStrategyHostAssisted, BlockMode),
)

DescribeTable("should create pvc clone pod with service mesh lifecycle hook", func(annotations map[string]string, expectedHook *v1.Lifecycle) {
dv := newCloneDataVolumeWithEmptyStorage("test-dv", "default")
cloneStrategy := cdiv1.CloneStrategyHostAssisted
storageProfile := createStorageProfileWithCloneStrategy(scName, []cdiv1.ClaimPropertySet{
{AccessModes: accessMode, VolumeMode: &BlockMode}}, &cloneStrategy)

maps.Copy(annotations, annKubevirt)
pvc := CreatePvcInStorageClass("test", metav1.NamespaceDefault, &scName, annotations, nil, corev1.ClaimBound)
reconciler := createCloneReconciler(dv, pvc, storageProfile, sc)
pod := reconciler.makeSizeDetectionPodSpec(pvc, dv)

By("Verifying the pod lifecycle hook is set")
Expect(pod.Spec.Containers[0].Lifecycle).To(Equal(expectedHook))
},
Entry("should create pod with linkerd mesh lifecycle hook", map[string]string{cc.AnnPodSidecarInjectionLinkerd: "enabled"}, &v1.Lifecycle{PreStop: linkerd.PreStopHook()}),
Entry("should create pod with istio mesh lifecycle hook", map[string]string{cc.AnnPodSidecarInjectionIstio: "true"}, &v1.Lifecycle{PreStop: istio.PreStopHook()}),
)
})
})

Expand Down
Loading

0 comments on commit 4d49377

Please sign in to comment.