Add envtest for testing controllers

Signed-off-by: Adrian Pedriza <[email protected]>

cmd/main.go

@@ -203,7 +203,6 @@ func main() {
 
 		if err = (&controlplane.K0sController{
 			Client:     mgr.GetClient(),
-			Scheme:     mgr.GetScheme(),
 			ClientSet:  clientSet,
 			RESTConfig: restConfig,
 		}).SetupWithManager(mgr); err != nil {

go.mod

@@ -5,18 +5,22 @@ go 1.22.0
 require (
 	github.com/cloudflare/cfssl v1.6.4
 	github.com/go-logr/logr v1.4.2
+	github.com/gobuffalo/flect v1.0.2
 	github.com/google/uuid v1.6.0
 	github.com/imdario/mergo v0.3.16
 	github.com/k0sproject/k0s v1.27.2-0.20230504131248-94378e521a29
 	github.com/k0sproject/rig v0.18.7
 	github.com/k0sproject/version v0.6.0
 	github.com/onsi/ginkgo/v2 v2.20.2
 	github.com/onsi/gomega v1.34.2
+	github.com/pkg/errors v0.9.1
 	github.com/stretchr/testify v1.9.0
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.28.4
+	k8s.io/apiextensions-apiserver v0.28.4
 	k8s.io/apimachinery v0.28.4
 	k8s.io/client-go v0.28.4
+	k8s.io/klog/v2 v2.100.1
 	k8s.io/kubernetes v1.28.4
 	k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0
 	sigs.k8s.io/controller-runtime v0.16.5
@@ -25,7 +29,7 @@ require (
 
 require (
 	github.com/Masterminds/goutils v1.1.1 // indirect
-	github.com/Masterminds/semver v1.5.0
+	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Masterminds/semver/v3 v3.2.1 // indirect
 	github.com/Masterminds/sprig v2.22.0+incompatible // indirect
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
@@ -56,7 +60,7 @@ require (
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	golang.org/x/crypto v0.27.0
 	golang.org/x/sync v0.8.0 // indirect
-	golang.org/x/tools v0.24.0 // indirect
+	golang.org/x/tools v0.24.0
 	gotest.tools/v3 v3.4.0 // indirect
 	helm.sh/helm/v3 v3.11.3 // indirect
 	k8s.io/kube-aggregator v0.27.2 // indirect
@@ -90,7 +94,6 @@ require (
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
-	github.com/gobuffalo/flect v1.0.2 // indirect
 	github.com/gofrs/uuid v4.4.0+incompatible // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -124,7 +127,6 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.17.0 // indirect
 	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
 	github.com/prometheus/common v0.44.0 // indirect
@@ -151,6 +153,7 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.25.0 // indirect
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
+	golang.org/x/mod v0.20.0 // indirect
 	golang.org/x/net v0.28.0 // indirect
 	golang.org/x/oauth2 v0.14.0 // indirect
 	golang.org/x/sys v0.25.0 // indirect
@@ -167,14 +170,12 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/apiextensions-apiserver v0.28.4 // indirect
 	k8s.io/apiserver v0.28.4 // indirect
 	k8s.io/cloud-provider v0.27.1 // indirect
 	k8s.io/cluster-bootstrap v0.28.4 // indirect
 	k8s.io/component-base v0.28.4 // indirect
 	k8s.io/component-helpers v0.28.4 // indirect
 	k8s.io/controller-manager v0.28.4 // indirect
-	k8s.io/klog/v2 v2.100.1 // indirect
 	k8s.io/kms v0.28.4 // indirect
 	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
 	k8s.io/kubelet v0.27.1 // indirect

internal/controller/controlplane/helper.go

@@ -29,7 +29,7 @@ func (c *K0sController) createMachine(ctx context.Context, name string, cluster
 	if err != nil {
 		return nil, fmt.Errorf("error generating machine: %w", err)
 	}
-	_ = ctrl.SetControllerReference(kcp, machine, c.Scheme)
+	_ = ctrl.SetControllerReference(kcp, machine, c.Client.Scheme())
 
 	return machine, c.Client.Patch(ctx, machine, client.Apply, &client.PatchOptions{
 		FieldManager: "k0smotron",
@@ -171,7 +171,7 @@ func (c *K0sController) generateMachineFromTemplate(ctx context.Context, name st
 		return nil, err
 	}
 
-	_ = ctrl.SetControllerReference(kcp, unstructuredMachineTemplate, c.Scheme)
+	_ = ctrl.SetControllerReference(kcp, unstructuredMachineTemplate, c.Client.Scheme())
 
 	template, found, err := unstructured.NestedMap(unstructuredMachineTemplate.UnstructuredContent(), "spec", "template")
 	if !found {

internal/controller/controlplane/k0s_controlplane_controller.go

@@ -31,7 +31,6 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -61,7 +60,6 @@ var ErrNewMachinesNotReady = fmt.Errorf("waiting for new machines")
 
 type K0sController struct {
 	client.Client
-	Scheme     *runtime.Scheme
 	ClientSet  *kubernetes.Clientset
 	RESTConfig *rest.Config
 }
@@ -116,6 +114,11 @@ func (c *K0sController) Reconcile(ctx context.Context, req ctrl.Request) (res ct
 		return ctrl.Result{}, nil
 	}
 
+	if annotations.IsPaused(cluster, kcp) {
+		log.Info("Reconciliation is paused for this object or owning cluster")
+		return ctrl.Result{}, nil
+	}
+
 	// Always patch the object to update the status
 	defer func() {
 		log.Info("Updating status")
@@ -147,11 +150,6 @@ func (c *K0sController) Reconcile(ctx context.Context, req ctrl.Request) (res ct
 
 	log = log.WithValues("cluster", cluster.Name)
 
-	if annotations.IsPaused(cluster, kcp) {
-		log.Info("Reconciliation is paused for this object or owning cluster")
-		return ctrl.Result{}, nil
-	}
-
 	if err := c.ensureCertificates(ctx, cluster, kcp); err != nil {
 		log.Error(err, "Failed to ensure certificates")
 		return ctrl.Result{}, err
@@ -620,7 +618,7 @@ token = ` + frpToken + `
 		},
 	}
 
-	_ = ctrl.SetControllerReference(kcp, &cm, c.Scheme)
+	_ = ctrl.SetControllerReference(kcp, &cm, c.Client.Scheme())
 	err = c.Client.Patch(ctx, &cm, client.Apply, &client.PatchOptions{FieldManager: "k0s-bootstrap"})
 	if err != nil {
 		return fmt.Errorf("error creating ConfigMap: %w", err)
@@ -689,7 +687,7 @@ token = ` + frpToken + `
 				}},
 		},
 	}
-	_ = ctrl.SetControllerReference(kcp, &frpsDeployment, c.Scheme)
+	_ = ctrl.SetControllerReference(kcp, &frpsDeployment, c.Client.Scheme())
 	err = c.Client.Patch(ctx, &frpsDeployment, client.Apply, &client.PatchOptions{FieldManager: "k0s-bootstrap"})
 	if err != nil {
 		return fmt.Errorf("error creating Deployment: %w", err)
@@ -725,7 +723,7 @@ token = ` + frpToken + `
 			Type: corev1.ServiceTypeNodePort,
 		},
 	}
-	_ = ctrl.SetControllerReference(kcp, &frpsService, c.Scheme)
+	_ = ctrl.SetControllerReference(kcp, &frpsService, c.Client.Scheme())
 	err = c.Client.Patch(ctx, &frpsService, client.Apply, &client.PatchOptions{FieldManager: "k0s-bootstrap"})
 	if err != nil {
 		return fmt.Errorf("error creating Service: %w", err)
@@ -773,7 +771,7 @@ func (c *K0sController) createFRPToken(ctx context.Context, cluster *clusterv1.C
 		Type: clusterv1.ClusterSecretType,
 	}
 
-	_ = ctrl.SetControllerReference(kcp, frpSecret, c.Scheme)
+	_ = ctrl.SetControllerReference(kcp, frpSecret, c.Client.Scheme())
 
 	return frpToken, c.Client.Patch(ctx, frpSecret, client.Apply, &client.PatchOptions{
 		FieldManager: "k0smotron",

internal/controller/controlplane/k0s_controlplane_controller_test.go

@@ -1,14 +1,25 @@
 package controlplane
 
 import (
+	"fmt"
 	"testing"
+	"time"
 
+	"github.com/imdario/mergo"
+	. "github.com/onsi/gomega"
 	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/types"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/util"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	bootstrapv1 "github.com/k0sproject/k0smotron/api/bootstrap/v1beta1"
 	"github.com/k0sproject/k0smotron/api/controlplane/v1beta1"
+	cpv1beta1 "github.com/k0sproject/k0smotron/api/controlplane/v1beta1"
 )
 
 func TestK0sConfigEnrichment(t *testing.T) {
@@ -103,3 +114,176 @@ func TestK0sConfigEnrichment(t *testing.T) {
 		})
 	}
 }
+
+func TestReconcileReturnErrorWhenOwnerClusterIsMissing(t *testing.T) {
+	g := NewWithT(t)
+
+	ns, err := testEnv.CreateNamespace(ctx, "test-reconcile-return-error")
+	g.Expect(err).ToNot(HaveOccurred())
+
+	cluster, kcp, gmt := createClusterWithControlPlane(ns.Name)
+	g.Expect(testEnv.Create(ctx, cluster)).To(Succeed())
+	g.Expect(testEnv.Create(ctx, kcp)).To(Succeed())
+	g.Expect(testEnv.Create(ctx, gmt)).To(Succeed())
+	defer func(do ...client.Object) {
+		g.Expect(testEnv.Cleanup(ctx, do...)).To(Succeed())
+	}(kcp, ns)
+
+	r := &K0sController{
+		Client: testEnv,
+	}
+
+	result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: util.ObjectKey(kcp)})
+	g.Expect(err).ToNot(HaveOccurred())
+	g.Expect(result).To(BeComparableTo(ctrl.Result{RequeueAfter: 20 * time.Second, Requeue: true}))
+
+	g.Expect(testEnv.CleanupAndWait(ctx, cluster)).To(Succeed())
+
+	g.Eventually(func() error {
+		_, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: util.ObjectKey(kcp)})
+		return err
+	}, 10*time.Second).Should(HaveOccurred())
+}
+
+func TestReconcileDoesNotNeedTakeAction(t *testing.T) {
+	g := NewWithT(t)
+	ns, err := testEnv.CreateNamespace(ctx, "test-reconcile-no-action-needed")
+	g.Expect(err).ToNot(HaveOccurred())
+
+	testCases := []struct {
+		name               string
+		controlplane       *cpv1beta1.K0sControlPlane
+		objRequestOverride metav1.Object
+	}{
+		{
+			name:         "K0sControlPlane not created",
+			controlplane: nil,
+			objRequestOverride: &cpv1beta1.K0sControlPlane{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "nonexisting",
+					Namespace: ns.Name,
+				},
+			},
+		},
+		{
+			name: "K0sControlPlane with paused annotation",
+			controlplane: &cpv1beta1.K0sControlPlane{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						"cluster.x-k8s.io/paused": "true",
+					},
+				},
+			},
+			objRequestOverride: nil,
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			cluster, kcp, _ := createClusterWithControlPlane(ns.Name)
+			g.Expect(testEnv.Create(ctx, cluster)).To(Succeed())
+
+			if tc.controlplane != nil {
+				err := mergo.Merge(kcp, tc.controlplane, mergo.WithOverride)
+				g.Expect(err).ToNot(HaveOccurred())
+			}
+			g.Expect(testEnv.Create(ctx, kcp)).To(Succeed())
+
+			defer func(do ...client.Object) {
+				g.Expect(testEnv.Cleanup(ctx, do...)).To(Succeed())
+			}(kcp, cluster)
+
+			r := &K0sController{
+				Client: testEnv,
+			}
+
+			or := tc.objRequestOverride
+			if tc.objRequestOverride == nil {
+				or = kcp
+			}
+			result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: util.ObjectKey(or)})
+			g.Expect(err).ToNot(HaveOccurred())
+			g.Expect(result).To(BeComparableTo(ctrl.Result{}))
+		})
+	}
+
+}
+
+func newCluster(namespacedName *types.NamespacedName) *clusterv1.Cluster {
+	return &clusterv1.Cluster{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Cluster",
+			APIVersion: clusterv1.GroupVersion.String(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: namespacedName.Namespace,
+			Name:      namespacedName.Name,
+		},
+	}
+}
+
+func createClusterWithControlPlane(namespace string) (*clusterv1.Cluster, *cpv1beta1.K0sControlPlane, *unstructured.Unstructured) {
+	kcpName := fmt.Sprintf("kcp-foo-%s", util.RandomString(6))
+
+	cluster := newCluster(&types.NamespacedName{Name: kcpName, Namespace: namespace})
+	cluster.Spec = clusterv1.ClusterSpec{
+		ControlPlaneRef: &corev1.ObjectReference{
+			Kind:       "K0sControlPlane",
+			Namespace:  namespace,
+			Name:       kcpName,
+			APIVersion: cpv1beta1.GroupVersion.String(),
+		},
+		ControlPlaneEndpoint: clusterv1.APIEndpoint{
+			Host: "test.endpoint",
+			Port: 0000,
+		},
+	}
+
+	kcp := &cpv1beta1.K0sControlPlane{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: cpv1beta1.GroupVersion.String(),
+			Kind:       "K0sControlPlane",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      kcpName,
+			Namespace: namespace,
+			OwnerReferences: []metav1.OwnerReference{
+				{
+					Kind:       "Cluster",
+					APIVersion: clusterv1.GroupVersion.String(),
+					Name:       kcpName,
+					UID:        "1",
+				},
+			},
+		},
+		Spec: v1beta1.K0sControlPlaneSpec{
+			MachineTemplate: &v1beta1.K0sControlPlaneMachineTemplate{
+				InfrastructureRef: corev1.ObjectReference{
+					Kind:       "GenericInfrastructureMachineTemplate",
+					Namespace:  namespace,
+					Name:       "infra-foo",
+					APIVersion: "infrastructure.cluster.x-k8s.io/v1beta1",
+				},
+			},
+			Replicas: int32(3),
+		},
+	}
+
+	genericMachineTemplate := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"kind":       "GenericInfrastructureMachineTemplate",
+			"apiVersion": "infrastructure.cluster.x-k8s.io/v1beta1",
+			"metadata": map[string]interface{}{
+				"name":      "infra-foo",
+				"namespace": namespace,
+			},
+			"spec": map[string]interface{}{
+				"template": map[string]interface{}{
+					"spec": map[string]interface{}{
+						"hello": "world",
+					},
+				},
+			},
+		},
+	}
+	return cluster, kcp, genericMachineTemplate
+}