From 2180a28324eec7026db6e953a3a962aab91d6480 Mon Sep 17 00:00:00 2001 From: Jan Bouska Date: Fri, 14 Jun 2024 14:16:25 +0200 Subject: [PATCH] Error handler --- Makefile | 2 +- api/v1alpha1/ctlog_types.go | 4 + api/v1alpha1/rekor_types.go | 4 + .../rhtas-operator.clusterserviceversion.yaml | 2 +- bundle/manifests/rhtas.redhat.com_ctlogs.yaml | 9 ++ bundle/manifests/rhtas.redhat.com_rekors.yaml | 9 ++ cmd/main.go | 1 + config/crd/bases/rhtas.redhat.com_ctlogs.yaml | 9 ++ config/crd/bases/rhtas.redhat.com_rekors.yaml | 9 ++ internal/apis/conditions_aware.go | 10 ++ internal/controller/common/action/action.go | 12 ++- .../controller/common/action/base_action.go | 50 ++++++++-- .../action/transitions/restart_on_error.go | 6 +- .../transitions/restart_on_error_test.go | 94 +++++++++++++++++++ .../action/transitions/to_create_phase.go | 9 ++ .../action/transitions/to_initialize.go | 9 ++ .../action/transitions/to_pending_phase.go | 9 ++ .../common/utils/kubernetes/common.go | 5 + .../common/utils/kubernetes/deployment.go | 7 +- internal/controller/constants/config.go | 5 +- internal/controller/constants/constants.go | 7 +- .../controller/ctlog/actions/constants.go | 4 +- .../controller/ctlog/actions/deployment.go | 47 ++++++++-- .../ctlog/actions/handle_fulcio_root.go | 32 ++++--- .../controller/ctlog/actions/handle_keys.go | 70 ++++++++------ .../controller/ctlog/actions/initialize.go | 21 ++++- .../controller/ctlog/actions/monitoring.go | 42 +++++++-- internal/controller/ctlog/actions/pending.go | 8 ++ internal/controller/ctlog/actions/rbac.go | 38 +++----- .../controller/ctlog/actions/resolve_tree.go | 20 ++-- .../ctlog/actions/resolve_tree_test.go | 7 +- .../controller/ctlog/actions/serverConfig.go | 56 ++++++++--- internal/controller/ctlog/actions/service.go | 38 +++++++- internal/controller/ctlog/ctlog_controller.go | 36 +++++-- .../ctlog/ctlog_error_handler_test.go | 7 +- internal/controller/ctlog/suite_test.go | 3 +- .../controller/ctlog/utils/ctlog_config.go | 4 +- .../controller/fulcio/actions/deployment.go | 18 +++- .../fulcio/actions/generate_cert.go | 40 ++++---- internal/controller/fulcio/actions/ingress.go | 14 ++- .../controller/fulcio/actions/initialize.go | 12 ++- .../controller/fulcio/actions/monitoring.go | 20 ++-- internal/controller/fulcio/actions/rbac.go | 20 ++-- internal/controller/fulcio/actions/service.go | 11 ++- .../controller/fulcio/actions/servrConfig.go | 15 ++- .../backfillRedis/backfill_redis_cronjob.go | 48 +++++++--- .../controller/rekor/actions/constants.go | 1 + .../controller/rekor/actions/initialize.go | 8 ++ internal/controller/rekor/actions/rbac.go | 41 ++++---- .../rekor/actions/redis/deployment.go | 36 +++++-- .../rekor/actions/redis/initialize.go | 10 +- .../controller/rekor/actions/redis/svc.go | 36 +++++-- .../rekor/actions/server/deployment.go | 48 +++++++--- .../rekor/actions/server/generate_signer.go | 58 ++++++------ .../rekor/actions/server/ingress.go | 46 +++++++-- .../rekor/actions/server/initialize.go | 11 ++- .../rekor/actions/server/monitoring.go | 58 +++++++----- .../controller/rekor/actions/server/pvc.go | 24 +++-- .../rekor/actions/server/resolve_pub_key.go | 38 ++++---- .../actions/server/resolve_pub_key_test.go | 15 +-- .../rekor/actions/server/resolve_tree.go | 14 ++- .../rekor/actions/server/resolve_tree_test.go | 7 +- .../rekor/actions/server/servrConfig.go | 42 +++++++-- .../rekor/actions/server/status_url.go | 13 ++- .../controller/rekor/actions/server/svc.go | 38 ++++++-- .../controller/rekor/actions/ui/deployment.go | 35 +++++-- .../controller/rekor/actions/ui/ingress.go | 33 ++++++- .../controller/rekor/actions/ui/initialize.go | 12 ++- internal/controller/rekor/actions/ui/svc.go | 36 +++++-- internal/controller/rekor/rekor_controller.go | 44 +++++++-- .../controller/rekor/rekor_controller_test.go | 17 ++-- .../rekor/rekor_error_handler_test.go | 61 ++++++------ .../controller/rekor/rekor_hot_update_test.go | 71 +++++++------- .../securesign/actions/ensure_ctlog.go | 15 ++- .../securesign/actions/ensure_fulcio.go | 15 ++- .../securesign/actions/ensure_rekor.go | 15 ++- .../securesign/actions/ensure_trillian.go | 15 ++- .../securesign/actions/ensure_tuf.go | 15 ++- .../securesign/actions/initialize_status.go | 8 ++ .../controller/securesign/actions/rbac.go | 22 +++-- .../actions/segment_backup_cronjob.go | 14 ++- .../securesign/actions/segment_backup_job.go | 12 ++- .../securesign/actions/update_status.go | 8 ++ .../trillian/actions/db/deployment.go | 13 ++- .../trillian/actions/db/handle_secret.go | 12 ++- .../trillian/actions/db/initialize.go | 10 +- .../controller/trillian/actions/db/pvc.go | 14 ++- .../controller/trillian/actions/db/svc.go | 12 ++- .../controller/trillian/actions/initialize.go | 8 ++ .../trillian/actions/logserver/deployment.go | 14 ++- .../trillian/actions/logserver/initialize.go | 10 +- .../trillian/actions/logserver/monitoring.go | 20 ++-- .../trillian/actions/logserver/service.go | 11 ++- .../trillian/actions/logsigner/deployment.go | 15 ++- .../trillian/actions/logsigner/initialize.go | 10 +- .../trillian/actions/logsigner/monitoring.go | 20 ++-- .../trillian/actions/logsigner/service.go | 11 ++- internal/controller/trillian/actions/rbac.go | 20 ++-- internal/controller/tuf/actions/deployment.go | 12 ++- .../controller/tuf/actions/generate_cert.go | 8 ++ internal/controller/tuf/actions/ingress.go | 16 +++- internal/controller/tuf/actions/initialize.go | 12 ++- internal/controller/tuf/actions/rbac.go | 20 ++-- internal/controller/tuf/actions/servise.go | 11 ++- internal/testing/action/result.go | 11 ++- test/e2e/support/common.go | 6 +- 106 files changed, 1607 insertions(+), 584 deletions(-) create mode 100644 internal/controller/common/action/transitions/restart_on_error_test.go diff --git a/Makefile b/Makefile index 1d9c3a418..4055f0a39 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,7 @@ endif # Set the Operator SDK version to use. By default, what is installed on the system is used. # This is useful for CI or a project to utilize a specific version of the operator-sdk toolkit. -OPERATOR_SDK_VERSION ?= v1.34.1 +OPERATOR_SDK_VERSION ?= v1.34.2 # Image URL to use all building/pushing image targets ifdef IMAGE_TAG diff --git a/api/v1alpha1/ctlog_types.go b/api/v1alpha1/ctlog_types.go index f1af7b356..eb688c96c 100644 --- a/api/v1alpha1/ctlog_types.go +++ b/api/v1alpha1/ctlog_types.go @@ -49,6 +49,9 @@ type CTlogStatus struct { RootCertificates []SecretKeySelector `json:"rootCertificates,omitempty"` // The ID of a Trillian tree that stores the log data. TreeID *int64 `json:"treeID,omitempty"` + // Number of component recovery attempts. + //+kubebuilder:default:=0 + RecoveryAttempts int64 `json:"recoveryAttempts,omitempty"` // +listType=map // +listMapKey=type // +patchStrategy=merge @@ -60,6 +63,7 @@ type CTlogStatus struct { //+kubebuilder:object:root=true //+kubebuilder:subresource:status //+kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].reason`,description="The component status" +//+kubebuilder:printcolumn:name="Recovery Attempts",type=string,JSONPath=`.status.recoveryAttempts`,description="The component recovery attempts" // CTlog is the Schema for the ctlogs API type CTlog struct { diff --git a/api/v1alpha1/rekor_types.go b/api/v1alpha1/rekor_types.go index 3564cae7b..123866a05 100644 --- a/api/v1alpha1/rekor_types.go +++ b/api/v1alpha1/rekor_types.go @@ -79,6 +79,9 @@ type RekorStatus struct { RekorSearchUIUrl string `json:"rekorSearchUIUrl,omitempty"` // The ID of a Trillian tree that stores the log data. TreeID *int64 `json:"treeID,omitempty"` + // Number of component recovery attempts. + //+kubebuilder:default:=0 + RecoveryAttempts int64 `json:"recoveryAttempts,omitempty"` // +listType=map // +listMapKey=type // +patchStrategy=merge @@ -91,6 +94,7 @@ type RekorStatus struct { //+kubebuilder:subresource:status //+kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].reason`,description="The component status" //+kubebuilder:printcolumn:name="URL",type=string,JSONPath=`.status.url`,description="The component url" +//+kubebuilder:printcolumn:name="Recovery Attempts",type=string,JSONPath=`.status.recoveryAttempts`,description="The component recovery attempts" // Rekor is the Schema for the rekors API type Rekor struct { diff --git a/bundle/manifests/rhtas-operator.clusterserviceversion.yaml b/bundle/manifests/rhtas-operator.clusterserviceversion.yaml index e57d02448..52d20415a 100644 --- a/bundle/manifests/rhtas-operator.clusterserviceversion.yaml +++ b/bundle/manifests/rhtas-operator.clusterserviceversion.yaml @@ -192,7 +192,7 @@ metadata: ] capabilities: Seamless Upgrades containerImage: registry.redhat.io/rhtas/rhtas-rhel9-operator@sha256:a21f7128694a64989bf0d84a7a7da4c1ffc89edf62d594dc8bea7bcfe9ac08d3 - createdAt: "2024-07-10T17:23:58Z" + createdAt: "2024-07-25T08:46:13Z" features.operators.openshift.io/cnf: "false" features.operators.openshift.io/cni: "false" features.operators.openshift.io/csi: "false" diff --git a/bundle/manifests/rhtas.redhat.com_ctlogs.yaml b/bundle/manifests/rhtas.redhat.com_ctlogs.yaml index deece3f75..2300b68e8 100644 --- a/bundle/manifests/rhtas.redhat.com_ctlogs.yaml +++ b/bundle/manifests/rhtas.redhat.com_ctlogs.yaml @@ -19,6 +19,10 @@ spec: jsonPath: .status.conditions[?(@.type=="Ready")].reason name: Status type: string + - description: The component recovery attempts + jsonPath: .status.recoveryAttempts + name: Recovery Attempts + type: string name: v1alpha1 schema: openAPIV3Schema: @@ -278,6 +282,11 @@ spec: - name type: object x-kubernetes-map-type: atomic + recoveryAttempts: + default: 0 + description: Number of component recovery attempts. + format: int64 + type: integer rootCertificates: items: description: SecretKeySelector selects a key of a Secret. diff --git a/bundle/manifests/rhtas.redhat.com_rekors.yaml b/bundle/manifests/rhtas.redhat.com_rekors.yaml index ae46216a7..e5ecb83d0 100644 --- a/bundle/manifests/rhtas.redhat.com_rekors.yaml +++ b/bundle/manifests/rhtas.redhat.com_rekors.yaml @@ -23,6 +23,10 @@ spec: jsonPath: .status.url name: URL type: string + - description: The component recovery attempts + jsonPath: .status.recoveryAttempts + name: Recovery Attempts + type: string name: v1alpha1 schema: openAPIV3Schema: @@ -321,6 +325,11 @@ spec: x-kubernetes-map-type: atomic pvcName: type: string + recoveryAttempts: + default: 0 + description: Number of component recovery attempts. + format: int64 + type: integer rekorSearchUIUrl: type: string serverConfigRef: diff --git a/cmd/main.go b/cmd/main.go index 877485e68..8e6cecec5 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -94,6 +94,7 @@ func main() { "If set, HTTP/2 will be enabled for the metrics and webhook servers") flag.Int64Var(&constants.CreateTreeDeadline, "create-tree-deadline", constants.CreateTreeDeadline, "The time allowance (in seconds) for the create tree job to run before failing.") utils.BoolFlagOrEnv(&constants.Openshift, "openshift", "OPENSHIFT", false, "Enable to ensures the operator applies OpenShift specific configurations.") + flag.Int64Var(&constants.AllowedRecoveryAttempts, "recovery-attempts", constants.AllowedRecoveryAttempts, "Specifies the maximum number of recovery attempts allowed before an operation is considered failed.") utils.StringFlagOrEnv(&constants.TrillianLogSignerImage, "trillian-log-signer-image", "TRILLIAN_LOG_SIGNER_IMAGE", constants.TrillianLogSignerImage, "The image used for trillian log signer.") utils.StringFlagOrEnv(&constants.TrillianServerImage, "trillian-log-server-image", "TRILLIAN_LOG_SERVER_IMAGE", constants.TrillianServerImage, "The image used for trillian log server.") utils.StringFlagOrEnv(&constants.TrillianDbImage, "trillian-db-image", "TRILLIAN_DB_IMAGE", constants.TrillianDbImage, "The image used for trillian's database.") diff --git a/config/crd/bases/rhtas.redhat.com_ctlogs.yaml b/config/crd/bases/rhtas.redhat.com_ctlogs.yaml index ca762bb52..0359b9ea5 100644 --- a/config/crd/bases/rhtas.redhat.com_ctlogs.yaml +++ b/config/crd/bases/rhtas.redhat.com_ctlogs.yaml @@ -19,6 +19,10 @@ spec: jsonPath: .status.conditions[?(@.type=="Ready")].reason name: Status type: string + - description: The component recovery attempts + jsonPath: .status.recoveryAttempts + name: Recovery Attempts + type: string name: v1alpha1 schema: openAPIV3Schema: @@ -278,6 +282,11 @@ spec: - name type: object x-kubernetes-map-type: atomic + recoveryAttempts: + default: 0 + description: Number of component recovery attempts. + format: int64 + type: integer rootCertificates: items: description: SecretKeySelector selects a key of a Secret. diff --git a/config/crd/bases/rhtas.redhat.com_rekors.yaml b/config/crd/bases/rhtas.redhat.com_rekors.yaml index e57733aa7..5257e5620 100644 --- a/config/crd/bases/rhtas.redhat.com_rekors.yaml +++ b/config/crd/bases/rhtas.redhat.com_rekors.yaml @@ -23,6 +23,10 @@ spec: jsonPath: .status.url name: URL type: string + - description: The component recovery attempts + jsonPath: .status.recoveryAttempts + name: Recovery Attempts + type: string name: v1alpha1 schema: openAPIV3Schema: @@ -321,6 +325,11 @@ spec: x-kubernetes-map-type: atomic pvcName: type: string + recoveryAttempts: + default: 0 + description: Number of component recovery attempts. + format: int64 + type: integer rekorSearchUIUrl: type: string serverConfigRef: diff --git a/internal/apis/conditions_aware.go b/internal/apis/conditions_aware.go index c01cd403b..2daffa777 100644 --- a/internal/apis/conditions_aware.go +++ b/internal/apis/conditions_aware.go @@ -1,6 +1,8 @@ package apis import ( + "github.com/securesign/operator/internal/controller/constants" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -11,3 +13,11 @@ type ConditionsAwareObject interface { GetConditions() []metav1.Condition SetCondition(newCondition metav1.Condition) } + +func IsError(obj ConditionsAwareObject) bool { + if obj != nil && meta.IsStatusConditionFalse(obj.GetConditions(), constants.Ready) { + return meta.FindStatusCondition(obj.GetConditions(), constants.Ready).Reason == constants.Error + } else { + return false + } +} diff --git a/internal/controller/common/action/action.go b/internal/controller/common/action/action.go index 16db3b58f..d03253ae7 100644 --- a/internal/controller/common/action/action.go +++ b/internal/controller/common/action/action.go @@ -20,12 +20,18 @@ type Action[T apis.ConditionsAwareObject] interface { InjectRecorder(recorder record.EventRecorder) InjectLogger(logger logr.Logger) - // a user friendly name for the action + // Name a user friendly name for the action Name() string - // returns true if the action can handle the integration + // CanHandle returns true if the action can handle CanHandle(context.Context, T) bool - // executes the handling function + // Handle executes the handling function Handle(context.Context, T) *Result + + // CanHandleError returns true if the action can handle the error + CanHandleError(context.Context, T) bool + + // HandleError executes the error handling function for specific action + HandleError(context.Context, T) *Result } diff --git a/internal/controller/common/action/base_action.go b/internal/controller/common/action/base_action.go index 84c5687fd..e1225f880 100644 --- a/internal/controller/common/action/base_action.go +++ b/internal/controller/common/action/base_action.go @@ -8,11 +8,14 @@ import ( "strings" "time" - "github.com/securesign/operator/internal/controller/annotations" - "github.com/go-logr/logr" + "github.com/securesign/operator/internal/apis" + "github.com/securesign/operator/internal/controller/annotations" + "github.com/securesign/operator/internal/controller/constants" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" client2 "sigs.k8s.io/controller-runtime/pkg/client" @@ -49,21 +52,52 @@ func (action *BaseAction) StatusUpdate(ctx context.Context, obj client2.Object) if strings.Contains(err.Error(), OptimisticLockErrorMsg) { return &Result{Result: reconcile.Result{RequeueAfter: 1 * time.Second}, Err: nil} } - return action.Failed(err) + return action.Error(err) } // Requeue will be caused by update return &Result{Result: reconcile.Result{Requeue: false}} } -func (action *BaseAction) Failed(err error) *Result { +func (action *BaseAction) Error(err error) *Result { action.Logger.Error(err, "error during action execution") return &Result{ - Result: reconcile.Result{RequeueAfter: time.Duration(5) * time.Second}, - Err: err, + Err: err, + } +} + +// ErrorWithStatusUpdate - Set `Error` status on deployment and execute error-recovery loop in 10 second +func (action *BaseAction) ErrorWithStatusUpdate(ctx context.Context, err error, instance apis.ConditionsAwareObject) *Result { + action.Recorder.Event(instance, v1.EventTypeWarning, constants.Error, err.Error()) + + instance.SetCondition(metav1.Condition{ + Type: constants.Ready, + Status: metav1.ConditionFalse, + Reason: constants.Error, + Message: err.Error(), + }) + + if e := action.Client.Status().Update(ctx, instance); e != nil { + if strings.Contains(err.Error(), OptimisticLockErrorMsg) { + return &Result{Result: reconcile.Result{RequeueAfter: 1 * time.Second}, Err: err} + } + err = errors.Join(e, err) } + // Requeue is disabled for Error objects + // wait for 10 seconds and invoke error-handler + return &Result{Result: reconcile.Result{RequeueAfter: 10 * time.Second}} } -func (action *BaseAction) FailedWithStatusUpdate(ctx context.Context, err error, instance client2.Object) *Result { +// FailWithStatusUpdate - Throw deployment to the Failure state with no error-recovery attempts +func (action *BaseAction) FailWithStatusUpdate(ctx context.Context, err error, instance apis.ConditionsAwareObject) *Result { + action.Recorder.Event(instance, v1.EventTypeWarning, constants.Failure, err.Error()) + + instance.SetCondition(metav1.Condition{ + Type: constants.Ready, + Status: metav1.ConditionFalse, + Reason: constants.Failure, + Message: err.Error(), + }) + if e := action.Client.Status().Update(ctx, instance); e != nil { if strings.Contains(err.Error(), OptimisticLockErrorMsg) { return &Result{Result: reconcile.Result{RequeueAfter: 1 * time.Second}, Err: err} @@ -71,7 +105,7 @@ func (action *BaseAction) FailedWithStatusUpdate(ctx context.Context, err error, err = errors.Join(e, err) } // Requeue will be caused by update - return &Result{Result: reconcile.Result{Requeue: false}, Err: err} + return &Result{Result: reconcile.Result{Requeue: false}} } func (action *BaseAction) Return() *Result { diff --git a/internal/controller/common/action/transitions/restart_on_error.go b/internal/controller/common/action/transitions/restart_on_error.go index 3a06dc13d..57cc87888 100644 --- a/internal/controller/common/action/transitions/restart_on_error.go +++ b/internal/controller/common/action/transitions/restart_on_error.go @@ -21,7 +21,7 @@ type restartAction[T apis.ConditionsAwareObject] struct { } func (i restartAction[T]) Name() string { - return "restart deployment" + return "restart on error" } func (i restartAction[T]) CanHandle(_ context.Context, instance T) bool { @@ -54,11 +54,11 @@ func (i restartAction[T]) HandleError(ctx context.Context, instance T) *action.R if err != nil { return i.Error(err) } - if restarts < constants.AllowedRestarts { + if restarts < constants.AllowedRecoveryAttempts { instance.SetCondition(metav1.Condition{Type: constants.Ready, Status: metav1.ConditionFalse, Reason: constants.Pending}) } else { - return i.FailWithStatusUpdate(ctx, fmt.Errorf("restart threshold reached"), instance) + return i.FailWithStatusUpdate(ctx, fmt.Errorf("recovery threshold reached"), instance) } return i.StatusUpdate(ctx, instance) diff --git a/internal/controller/common/action/transitions/restart_on_error_test.go b/internal/controller/common/action/transitions/restart_on_error_test.go new file mode 100644 index 000000000..a11b2db3b --- /dev/null +++ b/internal/controller/common/action/transitions/restart_on_error_test.go @@ -0,0 +1,94 @@ +package transitions + +import ( + "context" + "fmt" + "testing" + + . "github.com/onsi/gomega" + "github.com/securesign/operator/api/v1alpha1" + "github.com/securesign/operator/internal/controller/constants" + testAction "github.com/securesign/operator/internal/testing/action" + "k8s.io/apimachinery/pkg/api/meta" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func Test_HandleError(t *testing.T) { + g := NewWithT(t) + + instance := &v1alpha1.CTlog{ + ObjectMeta: v1.ObjectMeta{Name: "error", Namespace: "default"}, + Status: v1alpha1.CTlogStatus{ + RecoveryAttempts: 0, + Conditions: []v1.Condition{{ + Type: constants.Ready, + Status: v1.ConditionFalse, + Reason: constants.Error, + }}, + }} + + a := NewRestartOnErrorAction[*v1alpha1.CTlog]() + f := testAction.FakeClientBuilder().WithObjects(instance).WithStatusSubresource(instance).Build() + a = testAction.PrepareAction(f, a) + + ctx := context.TODO() + g.Expect(a.CanHandleError(ctx, instance)).To(BeTrue()) + result := a.HandleError(ctx, instance) + + g.Expect(result).Should(Equal(testAction.StatusUpdate())) + g.Expect(instance.Status.RecoveryAttempts).Should(Equal(int64(1))) + g.Expect(meta.FindStatusCondition(instance.GetConditions(), constants.Ready).Reason).Should(Equal(constants.Pending)) +} + +func Test_HandleError_Threshold(t *testing.T) { + g := NewWithT(t) + + instance := &v1alpha1.CTlog{ + ObjectMeta: v1.ObjectMeta{Name: "treshold", Namespace: "default"}, + Status: v1alpha1.CTlogStatus{ + Conditions: []v1.Condition{{ + Type: constants.Ready, + Status: v1.ConditionFalse, + Reason: constants.Error, + }}, + RecoveryAttempts: constants.AllowedRecoveryAttempts - 1, + }} + + a := NewRestartOnErrorAction[*v1alpha1.CTlog]() + f := testAction.FakeClientBuilder().WithObjects(instance).WithStatusSubresource(instance).Build() + + a = testAction.PrepareAction(f, a) + ctx := context.TODO() + g.Expect(a.CanHandleError(ctx, instance)).To(BeTrue()) + result := a.HandleError(ctx, instance) + + g.Expect(result).Should(Equal(testAction.FailWithStatusUpdate(fmt.Errorf("error")))) + g.Expect(instance.Status.RecoveryAttempts).Should(Equal(constants.AllowedRecoveryAttempts)) + g.Expect(meta.FindStatusCondition(instance.GetConditions(), constants.Ready).Reason).Should(Equal(constants.Failure)) +} + +func Test_HandleError_Running(t *testing.T) { + g := NewWithT(t) + + instance := &v1alpha1.CTlog{ + ObjectMeta: v1.ObjectMeta{Name: "handleRunning", Namespace: "default"}, + Status: v1alpha1.CTlogStatus{ + Conditions: []v1.Condition{{ + Type: constants.Ready, + Status: v1.ConditionTrue, + Reason: constants.Ready, + }}, + RecoveryAttempts: 2, + }} + + a := NewRestartOnErrorAction[*v1alpha1.CTlog]() + f := testAction.FakeClientBuilder().WithObjects(instance).WithStatusSubresource(instance).Build() + a = testAction.PrepareAction(f, a) + ctx := context.TODO() + g.Expect(a.CanHandle(ctx, instance)).To(BeTrue()) + result := a.Handle(ctx, instance) + + g.Expect(result).Should(Equal(testAction.StatusUpdate())) + g.Expect(instance.Status.RecoveryAttempts).Should(Equal(int64(0))) + g.Expect(meta.FindStatusCondition(instance.GetConditions(), constants.Ready).Reason).Should(Equal(constants.Ready)) +} diff --git a/internal/controller/common/action/transitions/to_create_phase.go b/internal/controller/common/action/transitions/to_create_phase.go index 17b07c650..cf86ec0a1 100644 --- a/internal/controller/common/action/transitions/to_create_phase.go +++ b/internal/controller/common/action/transitions/to_create_phase.go @@ -31,3 +31,12 @@ func (i toCreate[T]) Handle(ctx context.Context, instance T) *action.Result { Status: metav1.ConditionFalse, Reason: constants.Creating}) return i.StatusUpdate(ctx, instance) } + +func (i toCreate[T]) CanHandleError(_ context.Context, _ T) bool { + return false +} + +func (i toCreate[T]) HandleError(_ context.Context, _ T) *action.Result { + // NO-OP + return i.Continue() +} diff --git a/internal/controller/common/action/transitions/to_initialize.go b/internal/controller/common/action/transitions/to_initialize.go index d8d96cab5..4c79597c3 100644 --- a/internal/controller/common/action/transitions/to_initialize.go +++ b/internal/controller/common/action/transitions/to_initialize.go @@ -33,3 +33,12 @@ func (i toInitializeAction[T]) Handle(ctx context.Context, instance T) *action.R return i.StatusUpdate(ctx, instance) } + +func (i toInitializeAction[T]) CanHandleError(_ context.Context, _ T) bool { + return false +} + +func (i toInitializeAction[T]) HandleError(_ context.Context, _ T) *action.Result { + // NO-OP + return i.Continue() +} diff --git a/internal/controller/common/action/transitions/to_pending_phase.go b/internal/controller/common/action/transitions/to_pending_phase.go index 5758c6f8a..06a19b7b3 100644 --- a/internal/controller/common/action/transitions/to_pending_phase.go +++ b/internal/controller/common/action/transitions/to_pending_phase.go @@ -40,3 +40,12 @@ func (i toPending[T]) Handle(ctx context.Context, instance T) *action.Result { } return i.StatusUpdate(ctx, instance) } + +func (i toPending[T]) CanHandleError(_ context.Context, _ T) bool { + return false +} + +func (i toPending[T]) HandleError(_ context.Context, _ T) *action.Result { + // NO-OP + return i.Continue() +} diff --git a/internal/controller/common/utils/kubernetes/common.go b/internal/controller/common/utils/kubernetes/common.go index 6cd5d0088..b566f1d47 100644 --- a/internal/controller/common/utils/kubernetes/common.go +++ b/internal/controller/common/utils/kubernetes/common.go @@ -50,6 +50,11 @@ func getDefaultKubeConfigFile() (string, error) { } func ContainerMode() (bool, error) { + if containerMode := os.Getenv("CONTAINER_MODE"); containerMode != "" { + if b, _ := strconv.ParseBool(containerMode); b { + return true, nil + } + } // When kube config is set, container mode is not used if os.Getenv(kubeConfigEnvVar) != "" { return false, nil diff --git a/internal/controller/common/utils/kubernetes/deployment.go b/internal/controller/common/utils/kubernetes/deployment.go index 9201105cc..3aa3070bc 100644 --- a/internal/controller/common/utils/kubernetes/deployment.go +++ b/internal/controller/common/utils/kubernetes/deployment.go @@ -10,17 +10,18 @@ import ( func DeploymentIsRunning(ctx context.Context, cli client.Client, namespace string, labels map[string]string) (bool, error) { var err error - list := &v1.DeploymentList{} + deploymentList := &v1.DeploymentList{} - if err = cli.List(ctx, list, client.InNamespace(namespace), client.MatchingLabels(labels)); err != nil { + if err = cli.List(ctx, deploymentList, client.InNamespace(namespace), client.MatchingLabels(labels)); err != nil { return false, err } - for _, d := range list.Items { + for _, d := range deploymentList.Items { c := getDeploymentCondition(d.Status, v1.DeploymentAvailable) if c == nil || c.Status == corev1.ConditionFalse { return false, nil } } + return true, nil } diff --git a/internal/controller/constants/config.go b/internal/controller/constants/config.go index e82022a79..308cf23df 100644 --- a/internal/controller/constants/config.go +++ b/internal/controller/constants/config.go @@ -1,6 +1,7 @@ package constants var ( - CreateTreeDeadline int64 = 1200 - Openshift bool + Openshift bool + CreateTreeDeadline int64 = 5 + AllowedRecoveryAttempts int64 = 100 ) diff --git a/internal/controller/constants/constants.go b/internal/controller/constants/constants.go index bf05286a3..5bef88939 100644 --- a/internal/controller/constants/constants.go +++ b/internal/controller/constants/constants.go @@ -8,5 +8,10 @@ const ( Pending = "Pending" Creating = "Creating" Initialize = "Initialize" - Failure = "Failure" + // The operator has encountered a recoverable error and is attempting to recover. This state includes a counter (RecoveryAttempts) to track the number of recovery attempts. + Recovering = "Recovering" + // The operator has encountered an unrecoverable error or exceeded the maximum number of recovery attempts. Reconciliation stops. + Failure = "Failure" + // An error has occurred, but it’s not yet classified as recoverable or irrecoverable. This state can transition to either Recovering or Failed based on error type and policy. + Error = "Error" ) diff --git a/internal/controller/ctlog/actions/constants.go b/internal/controller/ctlog/actions/constants.go index 78acb28fa..81cc45e58 100644 --- a/internal/controller/ctlog/actions/constants.go +++ b/internal/controller/ctlog/actions/constants.go @@ -6,10 +6,12 @@ const ( RBACName = "ctlog" MonitoringRoleName = "prometheus-k8s-ctlog" - CertCondition = "FulcioCertAvailable" ServerPortName = "http" ServerPort = 80 ServerTargetPort = 6962 MetricsPortName = "metrics" MetricsPort = 6963 + KeyCondition = "KeyAvailable" + CertCondition = "FulcioCertAvailable" + ServerCondition = "ServerAvailable" ) diff --git a/internal/controller/ctlog/actions/deployment.go b/internal/controller/ctlog/actions/deployment.go index c8563e424..12c6ee4b6 100644 --- a/internal/controller/ctlog/actions/deployment.go +++ b/internal/controller/ctlog/actions/deployment.go @@ -5,6 +5,10 @@ import ( "fmt" cutils "github.com/securesign/operator/internal/controller/common/utils" + v1 "k8s.io/api/apps/v1" + v12 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1" "github.com/securesign/operator/internal/controller/common/action" @@ -43,37 +47,64 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.CTlog) dp, err := utils.CreateDeployment(instance, DeploymentName, RBACName, labels, ServerTargetPort, MetricsPort) if err != nil { meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, + Type: ServerCondition, Status: metav1.ConditionFalse, Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could create server Deployment: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could create server Deployment: %w", err), instance) } err = cutils.SetTrustedCA(&dp.Spec.Template, cutils.TrustedCAAnnotationToReference(instance.Annotations)) if err != nil { - return i.Failed(err) + return i.Error(err) } if err = controllerutil.SetControllerReference(instance, dp, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for Deployment: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for Deployment: %w", err)) } if updated, err = i.Ensure(ctx, dp); err != nil { meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, + Type: ServerCondition, Status: metav1.ConditionFalse, Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create CTlog: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create CTlog: %w", err), instance) } if updated { - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{Type: constants.Ready, - Status: metav1.ConditionFalse, Reason: constants.Creating, Message: "Service created"}) + meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ + Type: ServerCondition, + Status: metav1.ConditionFalse, + Reason: constants.Creating, + Message: "Deployment created", + }) + i.Recorder.Eventf(instance, v12.EventTypeNormal, "DeploymentUpdated", "Deployment updated: %s", instance.Name) return i.StatusUpdate(ctx, instance) } else { return i.Continue() } } + +func (i deployAction) CanHandleError(ctx context.Context, instance *rhtasv1alpha1.CTlog) bool { + err := i.Client.Get(ctx, types.NamespacedName{Name: DeploymentName, Namespace: instance.Namespace}, &v1.Deployment{}) + return !meta.IsStatusConditionTrue(instance.GetConditions(), ServerCondition) && err == nil || !errors.IsNotFound(err) +} + +func (i deployAction) HandleError(ctx context.Context, instance *rhtasv1alpha1.CTlog) *action.Result { + redisDeployment := &v1.Deployment{} + if err := i.Client.Get(ctx, types.NamespacedName{Name: DeploymentName, Namespace: instance.Namespace}, redisDeployment); err != nil { + return i.Error(err) + } + if err := i.Client.Delete(ctx, redisDeployment); err != nil { + i.Logger.V(1).Info("Can't delete CTLog deployment", "error", err.Error()) + } + meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ + Type: ServerCondition, + Status: metav1.ConditionFalse, + Reason: constants.Recovering, + Message: "server deployment will be recreated", + }) + return i.StatusUpdate(ctx, instance) +} diff --git a/internal/controller/ctlog/actions/handle_fulcio_root.go b/internal/controller/ctlog/actions/handle_fulcio_root.go index 672d61a8c..dc1555015 100644 --- a/internal/controller/ctlog/actions/handle_fulcio_root.go +++ b/internal/controller/ctlog/actions/handle_fulcio_root.go @@ -4,7 +4,7 @@ import ( "context" "slices" - "github.com/securesign/operator/api/v1alpha1" + rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1" "github.com/securesign/operator/internal/controller/common/action" k8sutils "github.com/securesign/operator/internal/controller/common/utils/kubernetes" "github.com/securesign/operator/internal/controller/constants" @@ -16,7 +16,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func NewHandleFulcioCertAction() action.Action[*v1alpha1.CTlog] { +func NewHandleFulcioCertAction() action.Action[*rhtasv1alpha1.CTlog] { return &handleFulcioCert{} } @@ -28,7 +28,7 @@ func (g handleFulcioCert) Name() string { return "handle-fulcio-cert" } -func (g handleFulcioCert) CanHandle(ctx context.Context, instance *v1alpha1.CTlog) bool { +func (g handleFulcioCert) CanHandle(ctx context.Context, instance *rhtasv1alpha1.CTlog) bool { c := meta.FindStatusCondition(instance.GetConditions(), constants.Ready) if c.Reason != constants.Creating && c.Reason != constants.Ready { return false @@ -45,8 +45,8 @@ func (g handleFulcioCert) CanHandle(ctx context.Context, instance *v1alpha1.CTlo if len(instance.Spec.RootCertificates) == 0 { // test if autodiscovery find new secret if scr, _ := k8sutils.FindSecret(ctx, g.Client, instance.Namespace, actions.FulcioCALabel); scr != nil { - return !slices.Contains(instance.Status.RootCertificates, v1alpha1.SecretKeySelector{ - LocalObjectReference: v1alpha1.LocalObjectReference{Name: scr.Name}, + return !slices.Contains(instance.Status.RootCertificates, rhtasv1alpha1.SecretKeySelector{ + LocalObjectReference: rhtasv1alpha1.LocalObjectReference{Name: scr.Name}, Key: scr.Labels[actions.FulcioCALabel], }) } @@ -55,7 +55,7 @@ func (g handleFulcioCert) CanHandle(ctx context.Context, instance *v1alpha1.CTlo return false } -func (g handleFulcioCert) Handle(ctx context.Context, instance *v1alpha1.CTlog) *action.Result { +func (g handleFulcioCert) Handle(ctx context.Context, instance *rhtasv1alpha1.CTlog) *action.Result { if meta.FindStatusCondition(instance.Status.Conditions, constants.Ready).Reason != constants.Creating { meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ @@ -71,9 +71,10 @@ func (g handleFulcioCert) Handle(ctx context.Context, instance *v1alpha1.CTlog) scr, err := k8sutils.FindSecret(ctx, g.Client, instance.Namespace, actions.FulcioCALabel) if err != nil { if !k8sErrors.IsNotFound(err) { - return g.Failed(err) + return g.Error(err) } - + } + if scr == nil { meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ Type: CertCondition, Status: metav1.ConditionFalse, @@ -83,9 +84,9 @@ func (g handleFulcioCert) Handle(ctx context.Context, instance *v1alpha1.CTlog) g.StatusUpdate(ctx, instance) return g.Requeue() } - instance.Status.RootCertificates = []v1alpha1.SecretKeySelector{ + instance.Status.RootCertificates = []rhtasv1alpha1.SecretKeySelector{ { - LocalObjectReference: v1alpha1.LocalObjectReference{ + LocalObjectReference: rhtasv1alpha1.LocalObjectReference{ Name: scr.Name, }, Key: scr.Labels[actions.FulcioCALabel], @@ -104,7 +105,7 @@ func (g handleFulcioCert) Handle(ctx context.Context, instance *v1alpha1.CTlog) }, }); err != nil { if !k8sErrors.IsNotFound(err) { - return g.Failed(err) + return g.Error(err) } } instance.Status.ServerConfigRef = nil @@ -118,3 +119,12 @@ func (g handleFulcioCert) Handle(ctx context.Context, instance *v1alpha1.CTlog) ) return g.StatusUpdate(ctx, instance) } + +func (i handleFulcioCert) CanHandleError(_ context.Context, instance *rhtasv1alpha1.CTlog) bool { + return len(instance.Status.RootCertificates) > 0 && !meta.IsStatusConditionTrue(instance.GetConditions(), CertCondition) +} + +func (i handleFulcioCert) HandleError(ctx context.Context, instance *rhtasv1alpha1.CTlog) *action.Result { + instance.Status.RootCertificates = nil + return i.StatusUpdate(ctx, instance) +} diff --git a/internal/controller/ctlog/actions/handle_keys.go b/internal/controller/ctlog/actions/handle_keys.go index d52aaaf19..2f0e268b5 100644 --- a/internal/controller/ctlog/actions/handle_keys.go +++ b/internal/controller/ctlog/actions/handle_keys.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/securesign/operator/api/v1alpha1" + rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1" "github.com/securesign/operator/internal/controller/common/action" k8sutils "github.com/securesign/operator/internal/controller/common/utils/kubernetes" "github.com/securesign/operator/internal/controller/constants" @@ -20,7 +20,7 @@ import ( const KeySecretNameFormat = "ctlog-%s-keys-" -func NewHandleKeysAction() action.Action[*v1alpha1.CTlog] { +func NewHandleKeysAction() action.Action[*rhtasv1alpha1.CTlog] { return &handleKeys{} } @@ -32,7 +32,7 @@ func (g handleKeys) Name() string { return "handle-keys" } -func (g handleKeys) CanHandle(ctx context.Context, instance *v1alpha1.CTlog) bool { +func (g handleKeys) CanHandle(ctx context.Context, instance *rhtasv1alpha1.CTlog) bool { c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready) if c.Reason != constants.Creating && c.Reason != constants.Ready { return false @@ -44,7 +44,7 @@ func (g handleKeys) CanHandle(ctx context.Context, instance *v1alpha1.CTlog) boo !equality.Semantic.DeepDerivative(instance.Spec.PrivateKeyPasswordRef, instance.Status.PublicKeyRef) } -func (g handleKeys) Handle(ctx context.Context, instance *v1alpha1.CTlog) *action.Result { +func (g handleKeys) Handle(ctx context.Context, instance *rhtasv1alpha1.CTlog) *action.Result { if meta.FindStatusCondition(instance.Status.Conditions, constants.Ready).Reason != constants.Creating { meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ Type: constants.Ready, @@ -61,7 +61,7 @@ func (g handleKeys) Handle(ctx context.Context, instance *v1alpha1.CTlog) *actio if instance.Spec.PrivateKeyRef == nil { config, err := utils.CreatePrivateKey() if err != nil { - return g.Failed(err) + return g.Error(err) } data = map[string][]byte{ "private": config.PrivateKey, @@ -77,7 +77,7 @@ func (g handleKeys) Handle(ctx context.Context, instance *v1alpha1.CTlog) *actio private, err = k8sutils.GetSecretData(g.Client, instance.Namespace, instance.Spec.PrivateKeyRef) if err != nil { meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, + Type: KeyCondition, Status: metav1.ConditionFalse, Reason: constants.Pending, Message: "Waiting for secret " + instance.Spec.PrivateKeyRef.Name, @@ -90,8 +90,8 @@ func (g handleKeys) Handle(ctx context.Context, instance *v1alpha1.CTlog) *actio password, err = k8sutils.GetSecretData(g.Client, instance.Namespace, instance.Spec.PrivateKeyPasswordRef) if err != nil { meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: constants.Creating, + Type: KeyCondition, + Status: metav1.ConditionFalse, Reason: constants.Pending, Message: "Waiting for secret " + instance.Spec.PrivateKeyPasswordRef.Name, }) @@ -102,7 +102,7 @@ func (g handleKeys) Handle(ctx context.Context, instance *v1alpha1.CTlog) *actio } config, err = utils.GeneratePublicKey(&utils.PrivateKeyConfig{PrivateKey: private, PrivateKeyPass: password}) if err != nil || config == nil { - return g.Failed(fmt.Errorf("unable to generate public key: %w", err)) + return g.Error(fmt.Errorf("unable to generate public key: %w", err)) } data = map[string][]byte{"public": config.PublicKey} } @@ -113,28 +113,22 @@ func (g handleKeys) Handle(ctx context.Context, instance *v1alpha1.CTlog) *actio data, labels) if err := controllerutil.SetControllerReference(instance, secret, g.Client.Scheme()); err != nil { - return g.Failed(fmt.Errorf("could not set controller reference for Secret: %w", err)) + return g.Error(fmt.Errorf("could not set controller reference for Secret: %w", err)) } // ensure that only new key is exposed if err := g.Client.DeleteAllOf(ctx, &v1.Secret{}, client.InNamespace(instance.Namespace), client.MatchingLabels(constants.LabelsFor(ComponentName, DeploymentName, instance.Name)), client.HasLabels{CTLPubLabel}); err != nil { - return g.Failed(err) + return g.Error(err) } if _, err := g.Ensure(ctx, secret); err != nil { - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return g.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create Secret: %w", err), instance) + return g.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create Secret: %w", err), instance) } if instance.Spec.PrivateKeyRef == nil { - instance.Status.PrivateKeyRef = &v1alpha1.SecretKeySelector{ + instance.Status.PrivateKeyRef = &rhtasv1alpha1.SecretKeySelector{ Key: "private", - LocalObjectReference: v1alpha1.LocalObjectReference{ + LocalObjectReference: rhtasv1alpha1.LocalObjectReference{ Name: secret.Name, }, } @@ -143,9 +137,9 @@ func (g handleKeys) Handle(ctx context.Context, instance *v1alpha1.CTlog) *actio } if _, ok := data["password"]; instance.Spec.PrivateKeyPasswordRef == nil && ok { - instance.Status.PrivateKeyPasswordRef = &v1alpha1.SecretKeySelector{ + instance.Status.PrivateKeyPasswordRef = &rhtasv1alpha1.SecretKeySelector{ Key: "password", - LocalObjectReference: v1alpha1.LocalObjectReference{ + LocalObjectReference: rhtasv1alpha1.LocalObjectReference{ Name: secret.Name, }, } @@ -154,9 +148,9 @@ func (g handleKeys) Handle(ctx context.Context, instance *v1alpha1.CTlog) *actio } if instance.Spec.PublicKeyRef == nil { - instance.Status.PublicKeyRef = &v1alpha1.SecretKeySelector{ + instance.Status.PublicKeyRef = &rhtasv1alpha1.SecretKeySelector{ Key: "public", - LocalObjectReference: v1alpha1.LocalObjectReference{ + LocalObjectReference: rhtasv1alpha1.LocalObjectReference{ Name: secret.Name, }, } @@ -173,17 +167,37 @@ func (g handleKeys) Handle(ctx context.Context, instance *v1alpha1.CTlog) *actio }, }); err != nil { if !k8sErrors.IsNotFound(err) { - return g.Failed(err) + return g.Error(err) } } instance.Status.ServerConfigRef = nil } meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Creating, + Type: KeyCondition, + Status: metav1.ConditionTrue, + Reason: constants.Ready, Message: "Keys resolved", }) return g.StatusUpdate(ctx, instance) } + +func (g handleKeys) CanHandleError(_ context.Context, instance *rhtasv1alpha1.CTlog) bool { + return !meta.IsStatusConditionTrue(instance.GetConditions(), KeyCondition) && + (instance.Status.PrivateKeyRef != nil || instance.Status.PublicKeyRef != nil || instance.Status.PrivateKeyPasswordRef != nil) +} + +func (g handleKeys) HandleError(ctx context.Context, instance *rhtasv1alpha1.CTlog) *action.Result { + instance.Status.PrivateKeyRef = nil + instance.Status.PublicKeyRef = nil + instance.Status.PrivateKeyPasswordRef = nil + + meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ + Type: KeyCondition, + Status: metav1.ConditionFalse, + Reason: constants.Recovering, + Message: "ctlog key will be recreated", + }) + + return g.StatusUpdate(ctx, instance) +} diff --git a/internal/controller/ctlog/actions/initialize.go b/internal/controller/ctlog/actions/initialize.go index 7180a71d2..a7f3490a8 100644 --- a/internal/controller/ctlog/actions/initialize.go +++ b/internal/controller/ctlog/actions/initialize.go @@ -36,7 +36,7 @@ func (i initializeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.CT labels := constants.LabelsForComponent(ComponentName, instance.Name) ok, err = commonUtils.DeploymentIsRunning(ctx, i.Client, instance.Namespace, labels) if err != nil { - return i.Failed(err) + return i.Error(err) } if !ok { i.Logger.Info("Waiting for deployment") @@ -46,6 +46,12 @@ func (i initializeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.CT Reason: constants.Initialize, Message: "Waiting for deployment to be ready", }) + meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ + Type: ServerCondition, + Status: metav1.ConditionFalse, + Reason: constants.Initialize, + Message: "Waiting for deployment to be ready", + }) return i.StatusUpdate(ctx, instance) } meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ @@ -53,5 +59,18 @@ func (i initializeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.CT Status: metav1.ConditionTrue, Reason: constants.Ready, }) + meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ + Type: ServerCondition, + Status: metav1.ConditionTrue, + Reason: constants.Ready, + }) return i.StatusUpdate(ctx, instance) } + +func (i initializeAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.CTlog) bool { + return false +} + +func (i initializeAction) HandleError(_ context.Context, _ *rhtasv1alpha1.CTlog) *action.Result { + return i.Continue() +} diff --git a/internal/controller/ctlog/actions/monitoring.go b/internal/controller/ctlog/actions/monitoring.go index 4e60affa7..2e5bc4c2a 100644 --- a/internal/controller/ctlog/actions/monitoring.go +++ b/internal/controller/ctlog/actions/monitoring.go @@ -10,8 +10,10 @@ import ( "github.com/securesign/operator/internal/controller/common/utils/kubernetes" "github.com/securesign/operator/internal/controller/constants" v1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -53,17 +55,17 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.CT ) if err = controllerutil.SetControllerReference(instance, role, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for role: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for role: %w", err)) } if _, err = i.Ensure(ctx, role); err != nil { meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, + Type: ServerCondition, Status: metav1.ConditionFalse, Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create monitoring role: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create monitoring role: %w", err), instance) } roleBinding := kubernetes.CreateRoleBinding( @@ -80,17 +82,17 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.CT }, ) if err = controllerutil.SetControllerReference(instance, roleBinding, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for role: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for role: %w", err)) } if _, err = i.Ensure(ctx, roleBinding); err != nil { meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, + Type: ServerCondition, Status: metav1.ConditionFalse, Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create monitoring RoleBinding: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create monitoring RoleBinding: %w", err), instance) } serviceMonitor := kubernetes.CreateServiceMonitor( @@ -108,19 +110,41 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.CT ) if err = controllerutil.SetControllerReference(instance, serviceMonitor, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for serviceMonitor: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for serviceMonitor: %w", err)) } if _, err = i.Ensure(ctx, serviceMonitor); err != nil { meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, + Type: ServerCondition, Status: metav1.ConditionFalse, Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create serviceMonitor: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create serviceMonitor: %w", err), instance) } // monitors & RBAC are not watched - do not need to re-enqueue return i.Continue() } + +func (i monitoringAction) CanHandleError(ctx context.Context, instance *rhtasv1alpha1.CTlog) bool { + err := i.Client.Get(ctx, types.NamespacedName{Name: DeploymentName, Namespace: instance.Namespace}, &monitoringv1.ServiceMonitor{}) + return !meta.IsStatusConditionTrue(instance.GetConditions(), ServerCondition) && instance.Spec.Monitoring.Enabled && (err == nil || !errors.IsNotFound(err)) +} + +func (i monitoringAction) HandleError(ctx context.Context, instance *rhtasv1alpha1.CTlog) *action.Result { + deployment := &monitoringv1.ServiceMonitor{} + if err := i.Client.Get(ctx, types.NamespacedName{Name: DeploymentName, Namespace: instance.Namespace}, deployment); err != nil { + return i.Error(err) + } + if err := i.Client.Delete(ctx, deployment); err != nil { + i.Logger.V(1).Info("Can't delete server deployment", "error", err.Error()) + } + meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ + Type: ServerCondition, + Status: metav1.ConditionFalse, + Reason: constants.Recovering, + Message: "servicemonitor will be recreated", + }) + return i.StatusUpdate(ctx, instance) +} diff --git a/internal/controller/ctlog/actions/pending.go b/internal/controller/ctlog/actions/pending.go index f5599a2ed..2ed956e7b 100644 --- a/internal/controller/ctlog/actions/pending.go +++ b/internal/controller/ctlog/actions/pending.go @@ -44,3 +44,11 @@ func (i pendingAction) Handle(ctx context.Context, instance *rhtasv1alpha1.CTlog } return i.Continue() } + +func (i pendingAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.CTlog) bool { + return false +} + +func (i pendingAction) HandleError(_ context.Context, _ *rhtasv1alpha1.CTlog) *action.Result { + return i.Continue() +} diff --git a/internal/controller/ctlog/actions/rbac.go b/internal/controller/ctlog/actions/rbac.go index f6a685f13..134c7e087 100644 --- a/internal/controller/ctlog/actions/rbac.go +++ b/internal/controller/ctlog/actions/rbac.go @@ -47,18 +47,12 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.CTlog) * } if err = ctrl.SetControllerReference(instance, sa, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controll reference for SA: %w", err)) + return i.Error(fmt.Errorf("could not set controll reference for SA: %w", err)) } // don't re-enqueue for RBAC in any case (except failure) _, err = i.Ensure(ctx, sa) if err != nil { - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create SA: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create SA: %w", err), instance) } role := kubernetes.CreateRole(instance.Namespace, RBACName, labels, []rbacv1.PolicyRule{ { @@ -74,17 +68,11 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.CTlog) * }) if err = ctrl.SetControllerReference(instance, role, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controll reference for role: %w", err)) + return i.Error(fmt.Errorf("could not set controll reference for role: %w", err)) } _, err = i.Ensure(ctx, role) if err != nil { - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create Role: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create Role: %w", err), instance) } rb := kubernetes.CreateRoleBinding(instance.Namespace, RBACName, labels, rbacv1.RoleRef{ APIGroup: v1.SchemeGroupVersion.Group, @@ -96,17 +84,19 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.CTlog) * }) if err = ctrl.SetControllerReference(instance, rb, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controll reference for roleBinding: %w", err)) + return i.Error(fmt.Errorf("could not set controll reference for roleBinding: %w", err)) } _, err = i.Ensure(ctx, rb) if err != nil { - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create RoleBinding: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create RoleBinding: %w", err), instance) } return i.Continue() } + +func (i rbacAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.CTlog) bool { + return false +} + +func (i rbacAction) HandleError(_ context.Context, _ *rhtasv1alpha1.CTlog) *action.Result { + return i.Continue() +} diff --git a/internal/controller/ctlog/actions/resolve_tree.go b/internal/controller/ctlog/actions/resolve_tree.go index 609910d0c..bf010b245 100644 --- a/internal/controller/ctlog/actions/resolve_tree.go +++ b/internal/controller/ctlog/actions/resolve_tree.go @@ -3,6 +3,7 @@ package actions import ( "context" "fmt" + "github.com/google/trillian" rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1" "github.com/securesign/operator/internal/controller/common" @@ -12,7 +13,6 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) type createTree func(ctx context.Context, displayName string, trillianURL string, deadline int64) (*trillian.Tree, error) @@ -65,16 +65,20 @@ func (i resolveTreeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.C tree, err = i.createTree(ctx, "ctlog-tree", trillUrl, constants.CreateTreeDeadline) if err != nil { - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create trillian tree: %v", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create trillian tree: %v", err), instance) } i.Recorder.Eventf(instance, v1.EventTypeNormal, "TrillianTreeCreated", "New Trillian tree created: %d", tree.TreeId) instance.Status.TreeID = &tree.TreeId return i.StatusUpdate(ctx, instance) } + +func (i resolveTreeAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.CTlog) bool { + // instance.Status.TreeID == nil in case of failure + // no need to recover + return false +} + +func (i resolveTreeAction) HandleError(_ context.Context, _ *rhtasv1alpha1.CTlog) *action.Result { + return i.Continue() +} diff --git a/internal/controller/ctlog/actions/resolve_tree_test.go b/internal/controller/ctlog/actions/resolve_tree_test.go index 5baa6df25..492ba84c1 100644 --- a/internal/controller/ctlog/actions/resolve_tree_test.go +++ b/internal/controller/ctlog/actions/resolve_tree_test.go @@ -4,6 +4,9 @@ import ( "context" "errors" "fmt" + "reflect" + "testing" + "github.com/google/trillian" . "github.com/onsi/gomega" rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1" @@ -14,8 +17,6 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" - "reflect" - "testing" ) func TestResolveTree_CanHandle(t *testing.T) { @@ -187,7 +188,7 @@ func TestResolveTree_Handle(t *testing.T) { createTree: mockCreateTree(nil, errors.New("timeout error"), nil), }, want: want{ - result: testAction.FailedWithStatusUpdate(fmt.Errorf("could not create trillian tree: timeout error")), + result: testAction.ErrorWithStatusUpdate(fmt.Errorf("could not create trillian tree: timeout error")), verify: func(g Gomega, rekor *rhtasv1alpha1.CTlog) { g.Expect(rekor.Spec.TreeID).Should(BeNil()) g.Expect(rekor.Status.TreeID).Should(BeNil()) diff --git a/internal/controller/ctlog/actions/serverConfig.go b/internal/controller/ctlog/actions/serverConfig.go index a927101ef..efe19a517 100644 --- a/internal/controller/ctlog/actions/serverConfig.go +++ b/internal/controller/ctlog/actions/serverConfig.go @@ -10,6 +10,7 @@ import ( utils "github.com/securesign/operator/internal/controller/common/utils/kubernetes" "github.com/securesign/operator/internal/controller/constants" ctlogUtils "github.com/securesign/operator/internal/controller/ctlog/utils" + "github.com/securesign/operator/internal/controller/rekor/actions" trillian "github.com/securesign/operator/internal/controller/trillian/actions" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -44,9 +45,9 @@ func (i serverConfig) Handle(ctx context.Context, instance *rhtasv1alpha1.CTlog) ) switch { case instance.Status.TreeID == nil: - return i.Failed(errors.New("reference to Trillian TreeID not set")) + return i.Error(errors.New("reference to Trillian TreeID not set")) case instance.Status.PrivateKeyRef == nil: - return i.Failed(errors.New("status reference to private key not set")) + return i.Error(errors.New("status reference to private key not set")) } labels := constants.LabelsFor(ComponentName, DeploymentName, instance.Name) @@ -65,7 +66,7 @@ func (i serverConfig) Handle(ctx context.Context, instance *rhtasv1alpha1.CTlog) rootCerts, err := i.handleRootCertificates(instance) if err != nil { - return i.Failed(err) + return i.Error(err) } certConfig, err := i.handlePrivateKey(instance) @@ -82,37 +83,35 @@ func (i serverConfig) Handle(ctx context.Context, instance *rhtasv1alpha1.CTlog) var cfg map[string][]byte if cfg, err = ctlogUtils.CreateCtlogConfig(trillUrl+":8091", *instance.Status.TreeID, rootCerts, certConfig); err != nil { - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create CTLog configuration: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create CTLog configuration: %w", err), instance) } newConfig := utils.CreateImmutableSecret(fmt.Sprintf("ctlog-config-%s", instance.Name), instance.Namespace, cfg, labels) if err = controllerutil.SetControllerReference(instance, newConfig, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for Secret: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for Secret: %w", err)) } _, err = i.Ensure(ctx, newConfig) if err != nil { meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, + Type: actions.ServerCondition, Status: metav1.ConditionFalse, Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, err, instance) + return i.ErrorWithStatusUpdate(ctx, err, instance) } instance.Status.ServerConfigRef = &rhtasv1alpha1.LocalObjectReference{Name: newConfig.Name} i.Recorder.Event(instance, corev1.EventTypeNormal, "CTLogConfigUpdated", "CTLog config updated") - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{Type: constants.Ready, - Status: metav1.ConditionFalse, Reason: constants.Creating, Message: "Server config created"}) + meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ + Type: ServerCondition, + Status: metav1.ConditionFalse, + Reason: constants.Creating, + Message: "Server config created", + }) return i.StatusUpdate(ctx, instance) } @@ -150,3 +149,30 @@ func (i serverConfig) handleRootCertificates(instance *rhtasv1alpha1.CTlog) ([]c return certs, nil } + +func (i serverConfig) CanHandleError(_ context.Context, instance *rhtasv1alpha1.CTlog) bool { + return !meta.IsStatusConditionTrue(instance.GetConditions(), actions.ServerCondition) && instance.Status.ServerConfigRef != nil +} + +func (i serverConfig) HandleError(ctx context.Context, instance *rhtasv1alpha1.CTlog) *action.Result { + deployment := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Status.ServerConfigRef.Name, + Namespace: instance.Namespace, + }, + } + if err := i.Client.Delete(ctx, deployment); err != nil { + i.Logger.V(1).Info("Can't delete server configuration", "error", err.Error()) + } + + instance.Status.ServerConfigRef = nil + + meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ + Type: ServerCondition, + Status: metav1.ConditionFalse, + Reason: constants.Recovering, + Message: "server configuration will be recreated", + }) + + return i.StatusUpdate(ctx, instance) +} diff --git a/internal/controller/ctlog/actions/service.go b/internal/controller/ctlog/actions/service.go index b1f35e895..71269563e 100644 --- a/internal/controller/ctlog/actions/service.go +++ b/internal/controller/ctlog/actions/service.go @@ -9,8 +9,10 @@ import ( "github.com/securesign/operator/internal/controller/common/utils/kubernetes" "github.com/securesign/operator/internal/controller/constants" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -50,24 +52,50 @@ func (i serviceAction) Handle(ctx context.Context, instance *rhtasv1alpha1.CTlog }) } if err = controllerutil.SetControllerReference(instance, svc, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for Service: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for Service: %w", err)) } if updated, err = i.Ensure(ctx, svc); err != nil { meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, + Type: ServerCondition, Status: metav1.ConditionFalse, Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create service: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create service: %w", err), instance) } if updated { - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{Type: constants.Ready, - Status: metav1.ConditionFalse, Reason: constants.Creating, Message: "Service created"}) + meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ + Type: ServerCondition, + Status: metav1.ConditionFalse, + Reason: constants.Creating, + Message: "Service created", + }) return i.StatusUpdate(ctx, instance) } else { return i.Continue() } } + +func (i serviceAction) CanHandleError(ctx context.Context, instance *rhtasv1alpha1.CTlog) bool { + err := i.Client.Get(ctx, types.NamespacedName{Name: DeploymentName, Namespace: instance.Namespace}, &corev1.Service{}) + return !meta.IsStatusConditionTrue(instance.GetConditions(), ServerCondition) && (err == nil || !errors.IsNotFound(err)) +} + +func (i serviceAction) HandleError(ctx context.Context, instance *rhtasv1alpha1.CTlog) *action.Result { + redisDeployment := &corev1.Service{} + if err := i.Client.Get(ctx, types.NamespacedName{Name: DeploymentName, Namespace: instance.Namespace}, redisDeployment); err != nil { + return i.Error(err) + } + if err := i.Client.Delete(ctx, redisDeployment); err != nil { + i.Logger.V(1).Info("Can't delete server service", "error", err.Error()) + } + meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ + Type: ServerCondition, + Status: metav1.ConditionFalse, + Reason: constants.Recovering, + Message: "server service will be recreated", + }) + return i.StatusUpdate(ctx, instance) +} diff --git a/internal/controller/ctlog/ctlog_controller.go b/internal/controller/ctlog/ctlog_controller.go index 5b7166206..7783b6f53 100644 --- a/internal/controller/ctlog/ctlog_controller.go +++ b/internal/controller/ctlog/ctlog_controller.go @@ -20,14 +20,15 @@ import ( "context" olpredicate "github.com/operator-framework/operator-lib/predicate" + "github.com/securesign/operator/internal/apis" "github.com/securesign/operator/internal/controller/annotations" "github.com/securesign/operator/internal/controller/common/action/transitions" - "k8s.io/apimachinery/pkg/runtime/schema" - + tasPredicate "github.com/securesign/operator/internal/controller/common/predicate" "github.com/securesign/operator/internal/controller/ctlog/actions" fulcioActions "github.com/securesign/operator/internal/controller/fulcio/actions" v12 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -90,7 +91,7 @@ func (r *CTlogReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl target := instance.DeepCopy() acs := []action.Action[*rhtasv1alpha1.CTlog]{ transitions.NewToPendingPhaseAction[*rhtasv1alpha1.CTlog](func(_ *rhtasv1alpha1.CTlog) []string { - return []string{actions.CertCondition} + return []string{actions.CertCondition, actions.KeyCondition, actions.ServerCondition} }), actions.NewPendingAction(), @@ -110,19 +111,31 @@ func (r *CTlogReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl transitions.NewToInitializePhaseAction[*rhtasv1alpha1.CTlog](), actions.NewInitializeAction(), + + // this should be always the last one + transitions.NewRestartOnErrorAction[*rhtasv1alpha1.CTlog](), } for _, a := range acs { - rlog.V(2).Info("Executing " + a.Name()) a.InjectClient(r.Client) a.InjectLogger(rlog.WithName(a.Name())) a.InjectRecorder(r.Recorder) - if a.CanHandle(ctx, target) { - rlog.V(1).Info("Executing " + a.Name()) - result := a.Handle(ctx, target) - if result != nil { - return result.Result, result.Err + if apis.IsError(&instance) { + if a.CanHandleError(ctx, target) { + rlog.V(2).Info("Executing error handling action " + a.Name()) + result := a.HandleError(ctx, target) + if result != nil { + return result.Result, result.Err + } + } + } else { + if a.CanHandle(ctx, target) { + rlog.V(2).Info("Executing " + a.Name()) + result := a.Handle(ctx, target) + if result != nil { + return result.Result, result.Err + } } } } @@ -156,7 +169,10 @@ func (r *CTlogReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). WithEventFilter(pause). - For(&rhtasv1alpha1.CTlog{}). + For(&rhtasv1alpha1.CTlog{}, builder.WithPredicates(predicate.And( + predicate.Or(predicate.GenerationChangedPredicate{}, tasPredicate.WaitOnError[*rhtasv1alpha1.CTlog]()), + tasPredicate.StopOnFailure[*rhtasv1alpha1.CTlog]()), + )). Owns(&v1.Deployment{}). Owns(&v12.Service{}). WatchesMetadata(partialSecret, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, object client.Object) []reconcile.Request { diff --git a/internal/controller/ctlog/ctlog_error_handler_test.go b/internal/controller/ctlog/ctlog_error_handler_test.go index 4bf7b12c8..965aad7f7 100644 --- a/internal/controller/ctlog/ctlog_error_handler_test.go +++ b/internal/controller/ctlog/ctlog_error_handler_test.go @@ -107,7 +107,7 @@ var _ = Describe("CTlog ErrorHandler", func() { map[string]string{fulcio.FulcioCALabel: "cert"}, ))).To(Succeed()) - Expect(k8sClient.Create(ctx, kubernetes.CreateService(Namespace, trillian.LogserverDeploymentName, trillian.ServerPortName, trillian.ServerPort, constants.LabelsForComponent(trillian.LogServerComponentName, instance.Name)))).To(Succeed()) + Expect(k8sClient.Create(ctx, kubernetes.CreateService(Namespace, trillian.LogserverDeploymentName, trillian.ServerPortName, trillian.ServerPort, trillian.ServerPort, constants.LabelsForComponent(trillian.LogServerComponentName, instance.Name)))).To(Succeed()) found := &v1alpha1.CTlog{} @@ -119,6 +119,7 @@ var _ = Describe("CTlog ErrorHandler", func() { if condition == nil { return "" } + return condition.Reason }).Should(Equal(constants.Error)) @@ -129,12 +130,12 @@ var _ = Describe("CTlog ErrorHandler", func() { Eventually(func(g Gomega) v1alpha1.CTlogStatus { g.Expect(k8sClient.Get(ctx, typeNamespaceName, found)).Should(Succeed()) return found.Status - }).WithPolling(1 * time.Second).WithTimeout(5 * time.Minute).Should(And( + }).Should(And( WithTransform( func(status v1alpha1.CTlogStatus) string { return meta.FindStatusCondition(status.Conditions, constants.Ready).Reason }, Not(Equal(constants.Error))), - WithTransform(func(status v1alpha1.CTlogStatus) int { + WithTransform(func(status v1alpha1.CTlogStatus) int64 { return status.RecoveryAttempts }, BeNumerically(">=", 1)))) Eventually(func(g Gomega) string { diff --git a/internal/controller/ctlog/suite_test.go b/internal/controller/ctlog/suite_test.go index ec3718da5..0bc259a2b 100644 --- a/internal/controller/ctlog/suite_test.go +++ b/internal/controller/ctlog/suite_test.go @@ -27,6 +27,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1" + "go.uber.org/zap/zapcore" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" @@ -56,7 +57,7 @@ func TestAPIs(t *testing.T) { } var _ = BeforeSuite(func() { - logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true), zap.Level(zapcore.Level(-2)))) ctx, cancel = context.WithCancel(context.TODO()) diff --git a/internal/controller/ctlog/utils/ctlog_config.go b/internal/controller/ctlog/utils/ctlog_config.go index 5434ddbb4..a37bca059 100644 --- a/internal/controller/ctlog/utils/ctlog_config.go +++ b/internal/controller/ctlog/utils/ctlog_config.go @@ -179,13 +179,13 @@ func CreateCtlogConfig(trillianUrl string, treeID int64, rootCerts []RootCertifi for _, cert := range rootCerts { if err = ctlogConfig.AddRootCertificate(cert); err != nil { - return nil, fmt.Errorf("Failed to add fulcio root: %v", err) + return nil, fmt.Errorf("Error to add fulcio root: %v", err) } } config, err := ctlogConfig.MarshalConfig() if err != nil { - return nil, fmt.Errorf("Failed to marshal ctlog config: %v", err) + return nil, fmt.Errorf("Error to marshal ctlog config: %v", err) } data := map[string][]byte{ diff --git a/internal/controller/fulcio/actions/deployment.go b/internal/controller/fulcio/actions/deployment.go index d0aea6334..030eace07 100644 --- a/internal/controller/fulcio/actions/deployment.go +++ b/internal/controller/fulcio/actions/deployment.go @@ -25,8 +25,8 @@ func (i deployAction) Name() string { return "deploy" } -func (i deployAction) CanHandle(_ context.Context, tuf *rhtasv1alpha1.Fulcio) bool { - c := meta.FindStatusCondition(tuf.Status.Conditions, constants.Ready) +func (i deployAction) CanHandle(_ context.Context, instance *rhtasv1alpha1.Fulcio) bool { + c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready) return c.Reason == constants.Creating || c.Reason == constants.Ready } @@ -54,12 +54,12 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fulcio Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could create server Deployment: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could create server Deployment: %w", err), instance) } } if err = controllerutil.SetControllerReference(instance, dp, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for Deployment: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for Deployment: %w", err)) } if updated, err = i.Ensure(ctx, dp); err != nil { @@ -69,7 +69,7 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fulcio Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create Fulcio: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create Fulcio: %w", err), instance) } if updated { @@ -80,3 +80,11 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fulcio return i.Continue() } } + +func (i deployAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Fulcio) bool { + return false +} + +func (i deployAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Fulcio) *action.Result { + return i.Continue() +} diff --git a/internal/controller/fulcio/actions/generate_cert.go b/internal/controller/fulcio/actions/generate_cert.go index f07c6c8a8..ae5c578a5 100644 --- a/internal/controller/fulcio/actions/generate_cert.go +++ b/internal/controller/fulcio/actions/generate_cert.go @@ -8,7 +8,7 @@ import ( "fmt" "maps" - "github.com/securesign/operator/api/v1alpha1" + rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1" "github.com/securesign/operator/internal/controller/common" "github.com/securesign/operator/internal/controller/common/action" k8sutils "github.com/securesign/operator/internal/controller/common/utils/kubernetes" @@ -26,7 +26,7 @@ const ( FulcioCALabel = constants.LabelNamespace + "/fulcio_v1.crt.pem" ) -func NewHandleCertAction() action.Action[*v1alpha1.Fulcio] { +func NewHandleCertAction() action.Action[*rhtasv1alpha1.Fulcio] { return &handleCert{} } @@ -38,13 +38,13 @@ func (g handleCert) Name() string { return "handle-cert" } -func (g handleCert) CanHandle(_ context.Context, instance *v1alpha1.Fulcio) bool { +func (g handleCert) CanHandle(_ context.Context, instance *rhtasv1alpha1.Fulcio) bool { c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready) return (c.Reason == constants.Pending || c.Reason == constants.Ready) && (instance.Status.Certificate == nil || !equality.Semantic.DeepDerivative(instance.Spec.Certificate, *instance.Status.Certificate)) } -func (g handleCert) Handle(ctx context.Context, instance *v1alpha1.Fulcio) *action.Result { +func (g handleCert) Handle(ctx context.Context, instance *rhtasv1alpha1.Fulcio) *action.Result { if meta.FindStatusCondition(instance.Status.Conditions, constants.Ready).Reason != constants.Pending { meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ Type: constants.Ready, @@ -68,7 +68,7 @@ func (g handleCert) Handle(ctx context.Context, instance *v1alpha1.Fulcio) *acti Reason: constants.Failure, Message: err.Error(), }) - return g.FailedWithStatusUpdate(ctx, err, instance) + return g.ErrorWithStatusUpdate(ctx, err, instance) } labels := constants.LabelsFor(ComponentName, DeploymentName, instance.Name) @@ -98,11 +98,11 @@ func (g handleCert) Handle(ctx context.Context, instance *v1alpha1.Fulcio) *acti newCert := k8sutils.CreateImmutableSecret(fmt.Sprintf("fulcio-cert-%s", instance.Name), instance.Namespace, cert.ToMap(), secretLabels) if err = controllerutil.SetControllerReference(instance, newCert, g.Client.Scheme()); err != nil { - return g.Failed(fmt.Errorf("could not set controller reference for Secret: %w", err)) + return g.Error(fmt.Errorf("could not set controller reference for Secret: %w", err)) } // ensure that only new key is exposed if err = g.Client.DeleteAllOf(ctx, &v1.Secret{}, client.InNamespace(instance.Namespace), client.MatchingLabels(constants.LabelsFor(ComponentName, DeploymentName, instance.Name)), client.HasLabels{FulcioCALabel}); err != nil { - return g.Failed(err) + return g.Error(err) } if _, err := g.Ensure(ctx, newCert); err != nil { meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ @@ -117,37 +117,37 @@ func (g handleCert) Handle(ctx context.Context, instance *v1alpha1.Fulcio) *acti Reason: constants.Failure, Message: err.Error(), }) - return g.FailedWithStatusUpdate(ctx, err, instance) + return g.ErrorWithStatusUpdate(ctx, err, instance) } g.Recorder.Event(instance, v1.EventTypeNormal, "FulcioCertUpdated", "Fulcio certificate secret updated") if instance.Status.Certificate == nil { - instance.Status.Certificate = new(v1alpha1.FulcioCert) + instance.Status.Certificate = new(rhtasv1alpha1.FulcioCert) } instance.Spec.Certificate.DeepCopyInto(instance.Status.Certificate) if instance.Spec.Certificate.PrivateKeyRef == nil { - instance.Status.Certificate.PrivateKeyRef = &v1alpha1.SecretKeySelector{ + instance.Status.Certificate.PrivateKeyRef = &rhtasv1alpha1.SecretKeySelector{ Key: "private", - LocalObjectReference: v1alpha1.LocalObjectReference{ + LocalObjectReference: rhtasv1alpha1.LocalObjectReference{ Name: newCert.Name, }, } } if instance.Spec.Certificate.PrivateKeyPasswordRef == nil && len(cert.PrivateKeyPassword) > 0 { - instance.Status.Certificate.PrivateKeyPasswordRef = &v1alpha1.SecretKeySelector{ + instance.Status.Certificate.PrivateKeyPasswordRef = &rhtasv1alpha1.SecretKeySelector{ Key: "password", - LocalObjectReference: v1alpha1.LocalObjectReference{ + LocalObjectReference: rhtasv1alpha1.LocalObjectReference{ Name: newCert.Name, }, } } if instance.Spec.Certificate.CARef == nil { - instance.Status.Certificate.CARef = &v1alpha1.SecretKeySelector{ + instance.Status.Certificate.CARef = &rhtasv1alpha1.SecretKeySelector{ Key: "cert", - LocalObjectReference: v1alpha1.LocalObjectReference{ + LocalObjectReference: rhtasv1alpha1.LocalObjectReference{ Name: newCert.Name, }, } @@ -161,7 +161,7 @@ func (g handleCert) Handle(ctx context.Context, instance *v1alpha1.Fulcio) *acti return g.StatusUpdate(ctx, instance) } -func (g handleCert) setupCert(ctx context.Context, instance *v1alpha1.Fulcio) (*utils.FulcioCertConfig, error) { +func (g handleCert) setupCert(ctx context.Context, instance *rhtasv1alpha1.Fulcio) (*utils.FulcioCertConfig, error) { config := &utils.FulcioCertConfig{} if ref := instance.Spec.Certificate.PrivateKeyPasswordRef; ref != nil { @@ -214,3 +214,11 @@ func (g handleCert) setupCert(ctx context.Context, instance *v1alpha1.Fulcio) (* return config, nil } + +func (i handleCert) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Fulcio) bool { + return false +} + +func (i handleCert) HandleError(_ context.Context, _ *rhtasv1alpha1.Fulcio) *action.Result { + return i.Continue() +} diff --git a/internal/controller/fulcio/actions/ingress.go b/internal/controller/fulcio/actions/ingress.go index 940ac6ceb..fad988752 100644 --- a/internal/controller/fulcio/actions/ingress.go +++ b/internal/controller/fulcio/actions/ingress.go @@ -40,16 +40,16 @@ func (i ingressAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fulci svc := &v1.Service{} if err := i.Client.Get(ctx, ok, svc); err != nil { - return i.Failed(fmt.Errorf("could not find service for ingress: %w", err)) + return i.Error(fmt.Errorf("could not find service for ingress: %w", err)) } ingress, err := kubernetes.CreateIngress(ctx, i.Client, *svc, instance.Spec.ExternalAccess, ServerPortName, labels) if err != nil { - return i.Failed(fmt.Errorf("could not create ingress object: %w", err)) + return i.Error(fmt.Errorf("could not create ingress object: %w", err)) } if err = controllerutil.SetControllerReference(instance, ingress, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for Ingress: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for Ingress: %w", err)) } if updated, err = i.Ensure(ctx, ingress); err != nil { @@ -69,3 +69,11 @@ func (i ingressAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fulci return i.Continue() } } + +func (i ingressAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Fulcio) bool { + return false +} + +func (i ingressAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Fulcio) *action.Result { + return i.Continue() +} diff --git a/internal/controller/fulcio/actions/initialize.go b/internal/controller/fulcio/actions/initialize.go index dd9e75e89..bb15f9cee 100644 --- a/internal/controller/fulcio/actions/initialize.go +++ b/internal/controller/fulcio/actions/initialize.go @@ -40,7 +40,7 @@ func (i initializeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fu labels := constants.LabelsForComponent(ComponentName, instance.Name) ok, err = commonUtils.DeploymentIsRunning(ctx, i.Client, instance.Namespace, labels) if err != nil { - return i.Failed(err) + return i.Error(err) } if !ok { i.Logger.Info("Waiting for deployment") @@ -58,7 +58,7 @@ func (i initializeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fu ingress := &v12.Ingress{} err = i.Client.Get(ctx, types.NamespacedName{Name: DeploymentName, Namespace: instance.Namespace}, ingress) if err != nil { - return i.Failed(err) + return i.Error(err) } if len(ingress.Spec.TLS) > 0 { protocol = "https://" @@ -72,3 +72,11 @@ func (i initializeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fu Status: metav1.ConditionTrue, Reason: constants.Ready}) return i.StatusUpdate(ctx, instance) } + +func (i initializeAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Fulcio) bool { + return false +} + +func (i initializeAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Fulcio) *action.Result { + return i.Continue() +} diff --git a/internal/controller/fulcio/actions/monitoring.go b/internal/controller/fulcio/actions/monitoring.go index 569eedc8a..712a2ca8c 100644 --- a/internal/controller/fulcio/actions/monitoring.go +++ b/internal/controller/fulcio/actions/monitoring.go @@ -53,7 +53,7 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fu ) if err = controllerutil.SetControllerReference(instance, role, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for role: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for role: %w", err)) } if _, err = i.Ensure(ctx, role); err != nil { @@ -63,7 +63,7 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fu Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create monitoring role: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create monitoring role: %w", err), instance) } roleBinding := kubernetes.CreateRoleBinding( @@ -80,7 +80,7 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fu }, ) if err = controllerutil.SetControllerReference(instance, roleBinding, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for role: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for role: %w", err)) } if _, err = i.Ensure(ctx, roleBinding); err != nil { @@ -90,7 +90,7 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fu Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create monitoring RoleBinding: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create monitoring RoleBinding: %w", err), instance) } serviceMonitor := kubernetes.CreateServiceMonitor( @@ -108,7 +108,7 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fu ) if err = controllerutil.SetControllerReference(instance, serviceMonitor, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for serviceMonitor: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for serviceMonitor: %w", err)) } if _, err = i.Ensure(ctx, serviceMonitor); err != nil { @@ -118,9 +118,17 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fu Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create serviceMonitor: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create serviceMonitor: %w", err), instance) } // monitors & RBAC are not watched - do not need to re-enqueue return i.Continue() } + +func (i monitoringAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Fulcio) bool { + return false +} + +func (i monitoringAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Fulcio) *action.Result { + return i.Continue() +} diff --git a/internal/controller/fulcio/actions/rbac.go b/internal/controller/fulcio/actions/rbac.go index cb99a5373..d94ebff32 100644 --- a/internal/controller/fulcio/actions/rbac.go +++ b/internal/controller/fulcio/actions/rbac.go @@ -47,7 +47,7 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fulcio) } if err = ctrl.SetControllerReference(instance, sa, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controll reference for SA: %w", err)) + return i.Error(fmt.Errorf("could not set controll reference for SA: %w", err)) } // don't re-enqueue for RBAC in any case (except failure) _, err = i.Ensure(ctx, sa) @@ -58,7 +58,7 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fulcio) Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create SA: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create SA: %w", err), instance) } role := kubernetes.CreateRole(instance.Namespace, RBACName, labels, []rbacv1.PolicyRule{ { @@ -74,7 +74,7 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fulcio) }) if err = ctrl.SetControllerReference(instance, role, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controll reference for role: %w", err)) + return i.Error(fmt.Errorf("could not set controll reference for role: %w", err)) } _, err = i.Ensure(ctx, role) if err != nil { @@ -84,7 +84,7 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fulcio) Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create Role: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create Role: %w", err), instance) } rb := kubernetes.CreateRoleBinding(instance.Namespace, RBACName, labels, rbacv1.RoleRef{ APIGroup: v1.SchemeGroupVersion.Group, @@ -96,7 +96,7 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fulcio) }) if err = ctrl.SetControllerReference(instance, rb, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controll reference for roleBinding: %w", err)) + return i.Error(fmt.Errorf("could not set controll reference for roleBinding: %w", err)) } _, err = i.Ensure(ctx, rb) if err != nil { @@ -106,7 +106,15 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fulcio) Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create RoleBinding: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create RoleBinding: %w", err), instance) } return i.Continue() } + +func (i rbacAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Fulcio) bool { + return false +} + +func (i rbacAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Fulcio) *action.Result { + return i.Continue() +} diff --git a/internal/controller/fulcio/actions/service.go b/internal/controller/fulcio/actions/service.go index b12a37af4..847596804 100644 --- a/internal/controller/fulcio/actions/service.go +++ b/internal/controller/fulcio/actions/service.go @@ -58,7 +58,7 @@ func (i serviceAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fulci } if err = controllerutil.SetControllerReference(instance, svc, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for Service: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for Service: %w", err)) } if updated, err = i.Ensure(ctx, svc); err != nil { meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ @@ -67,7 +67,7 @@ func (i serviceAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fulci Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create service: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create service: %w", err), instance) } if updated { @@ -77,5 +77,12 @@ func (i serviceAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Fulci } else { return i.Continue() } +} + +func (i serviceAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Fulcio) bool { + return false +} +func (i serviceAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Fulcio) *action.Result { + return i.Continue() } diff --git a/internal/controller/fulcio/actions/servrConfig.go b/internal/controller/fulcio/actions/servrConfig.go index fdf311ee8..4c336957f 100644 --- a/internal/controller/fulcio/actions/servrConfig.go +++ b/internal/controller/fulcio/actions/servrConfig.go @@ -81,13 +81,13 @@ func (i serverConfig) Handle(ctx context.Context, instance *rhtasv1alpha1.Fulcio config, err := json.Marshal(ConvertToFulcioMapConfig(instance.Spec.Config)) if err != nil { - return i.FailedWithStatusUpdate(ctx, err, instance) + return i.ErrorWithStatusUpdate(ctx, err, instance) } expected := kubernetes.CreateImmutableConfigmap(fmt.Sprintf("fulcio-config-%s", instance.Name), instance.Namespace, labels, map[string]string{ "config.json": string(config), }) if err = controllerutil.SetControllerReference(instance, expected, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for ConfigMap: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for ConfigMap: %w", err)) } // invalidate config @@ -98,7 +98,7 @@ func (i serverConfig) Handle(ctx context.Context, instance *rhtasv1alpha1.Fulcio Namespace: instance.Namespace, }, }); err != nil { - return i.Failed(err) + return i.Error(err) } instance.Status.ServerConfigRef = nil } @@ -116,7 +116,7 @@ func (i serverConfig) Handle(ctx context.Context, instance *rhtasv1alpha1.Fulcio Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, err, instance) + return i.ErrorWithStatusUpdate(ctx, err, instance) } instance.Status.ServerConfigRef = &rhtasv1alpha1.LocalObjectReference{Name: expected.Name} @@ -125,5 +125,12 @@ func (i serverConfig) Handle(ctx context.Context, instance *rhtasv1alpha1.Fulcio meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{Type: constants.Ready, Status: metav1.ConditionFalse, Reason: constants.Creating, Message: "Server config created"}) return i.StatusUpdate(ctx, instance) +} + +func (i serverConfig) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Fulcio) bool { + return false +} +func (i serverConfig) HandleError(_ context.Context, _ *rhtasv1alpha1.Fulcio) *action.Result { + return i.Continue() } diff --git a/internal/controller/rekor/actions/backfillRedis/backfill_redis_cronjob.go b/internal/controller/rekor/actions/backfillRedis/backfill_redis_cronjob.go index b5bafbd71..f5e8cd6ca 100644 --- a/internal/controller/rekor/actions/backfillRedis/backfill_redis_cronjob.go +++ b/internal/controller/rekor/actions/backfillRedis/backfill_redis_cronjob.go @@ -5,6 +5,8 @@ import ( "github.com/robfig/cron/v3" "github.com/securesign/operator/internal/controller/common/utils" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" "github.com/securesign/operator/internal/controller/constants" "github.com/securesign/operator/internal/controller/rekor/actions" @@ -45,7 +47,7 @@ func (i backfillRedisCronJob) Handle(ctx context.Context, instance *rhtasv1alpha ) if _, err := cron.ParseStandard(instance.Spec.BackFillRedis.Schedule); err != nil { - return i.Failed(fmt.Errorf("could not create backfill redis cron job: %w", err)) + return i.Error(fmt.Errorf("could not create backfill redis cron job: %w", err)) } labels := constants.LabelsFor(actions.BackfillRedisCronJobName, actions.BackfillRedisCronJobName, instance.Name) @@ -81,34 +83,52 @@ func (i backfillRedisCronJob) Handle(ctx context.Context, instance *rhtasv1alpha } if err = controllerutil.SetControllerReference(instance, backfillRedisCronJob, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for backfill redis cron job: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for backfill redis cron job: %w", err)) } if updated, err = i.Ensure(ctx, backfillRedisCronJob); err != nil { meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: actions.RedisCondition, + Type: actions.BackfillRedisCondition, Status: metav1.ConditionFalse, Reason: constants.Failure, Message: err.Error(), }) - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create backfill redis cron job: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create backfill redis cron job: %w", err), instance) } if updated { meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Creating, - Message: "Backfill redis job created", + Type: actions.BackfillRedisCondition, + Status: metav1.ConditionTrue, + Reason: constants.Ready, + Message: "Backfill created", }) return i.StatusUpdate(ctx, instance) } else { return i.Continue() } } + +func (i backfillRedisCronJob) CanHandleError(ctx context.Context, instance *rhtasv1alpha1.Rekor) bool { + err := i.Client.Get(ctx, types.NamespacedName{Name: actions.ServerDeploymentName, Namespace: instance.Namespace}, &batchv1.CronJob{}) + return utils.OptionalBool(instance.Spec.BackFillRedis.Enabled) && + !meta.IsStatusConditionTrue(instance.GetConditions(), actions.BackfillRedisCondition) && (err == nil || !errors.IsNotFound(err)) +} + +func (i backfillRedisCronJob) HandleError(ctx context.Context, instance *rhtasv1alpha1.Rekor) *action.Result { + bacfillCronJob := &batchv1.CronJob{} + if err := i.Client.Get(ctx, types.NamespacedName{Name: actions.BackfillRedisCronJobName, Namespace: instance.Namespace}, bacfillCronJob); err != nil { + return i.Error(err) + } + if err := i.Client.Delete(ctx, bacfillCronJob); err != nil { + i.Logger.V(1).Info("Can't delete BacfillCronJob", "error", err.Error()) + } + + meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ + Type: actions.BackfillRedisCondition, + Status: metav1.ConditionFalse, + Reason: constants.Recovering, + Message: "Backfill redis job will be recreated", + }) + return i.StatusUpdate(ctx, instance) +} diff --git a/internal/controller/rekor/actions/constants.go b/internal/controller/rekor/actions/constants.go index 359729279..9d3c52f21 100644 --- a/internal/controller/rekor/actions/constants.go +++ b/internal/controller/rekor/actions/constants.go @@ -23,4 +23,5 @@ const ( ServerCondition = "ServerAvailable" RedisCondition = "RedisAvailable" SignerCondition = "SignerAvailable" + BackfillRedisCondition = "BackfillRedisCronJobAvailable" ) diff --git a/internal/controller/rekor/actions/initialize.go b/internal/controller/rekor/actions/initialize.go index de54a34cc..5a113d8fa 100644 --- a/internal/controller/rekor/actions/initialize.go +++ b/internal/controller/rekor/actions/initialize.go @@ -48,3 +48,11 @@ func (i initializeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Re Status: metav1.ConditionTrue, Reason: constants.Ready}) return i.StatusUpdate(ctx, instance) } + +func (i initializeAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Rekor) bool { + return false +} + +func (i initializeAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Rekor) *action.Result { + return i.Continue() +} diff --git a/internal/controller/rekor/actions/rbac.go b/internal/controller/rekor/actions/rbac.go index 5d16ace52..47be1a121 100644 --- a/internal/controller/rekor/actions/rbac.go +++ b/internal/controller/rekor/actions/rbac.go @@ -29,6 +29,9 @@ func (i rbacAction) Name() string { func (i rbacAction) CanHandle(_ context.Context, instance *rhtasv1alpha1.Rekor) bool { c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready) + if c == nil { + return false + } return c.Reason == constants.Creating || c.Reason == constants.Ready } @@ -47,18 +50,12 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Rekor) * } if err = ctrl.SetControllerReference(instance, sa, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controll reference for SA: %w", err)) + return i.Error(fmt.Errorf("could not set controll reference for SA: %w", err)) } // don't re-enqueue for RBAC in any case (except failure) _, err = i.Ensure(ctx, sa) if err != nil { - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create SA: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create SA: %w", err), instance) } role := kubernetes.CreateRole(instance.Namespace, RBACName, labels, []rbacv1.PolicyRule{ { @@ -74,17 +71,11 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Rekor) * }) if err = ctrl.SetControllerReference(instance, role, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controll reference for role: %w", err)) + return i.Error(fmt.Errorf("could not set controll reference for role: %w", err)) } _, err = i.Ensure(ctx, role) if err != nil { - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create Role: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create Role: %w", err), instance) } rb := kubernetes.CreateRoleBinding(instance.Namespace, RBACName, labels, rbacv1.RoleRef{ APIGroup: v1.SchemeGroupVersion.Group, @@ -96,17 +87,19 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Rekor) * }) if err = ctrl.SetControllerReference(instance, rb, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controll reference for roleBinding: %w", err)) + return i.Error(fmt.Errorf("could not set controll reference for roleBinding: %w", err)) } _, err = i.Ensure(ctx, rb) if err != nil { - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create RoleBinding: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create RoleBinding: %w", err), instance) } return i.Continue() } + +func (i rbacAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Rekor) bool { + return false +} + +func (i rbacAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Rekor) *action.Result { + return i.Continue() +} diff --git a/internal/controller/rekor/actions/redis/deployment.go b/internal/controller/rekor/actions/redis/deployment.go index 57fef516e..f956d3c3f 100644 --- a/internal/controller/rekor/actions/redis/deployment.go +++ b/internal/controller/rekor/actions/redis/deployment.go @@ -8,8 +8,11 @@ import ( "github.com/securesign/operator/internal/controller/constants" "github.com/securesign/operator/internal/controller/rekor/actions" "github.com/securesign/operator/internal/controller/rekor/utils" + v1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1" @@ -40,7 +43,7 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Rekor) labels := constants.LabelsFor(actions.RedisComponentName, actions.RedisDeploymentName, instance.Name) dp := utils.CreateRedisDeployment(instance.Namespace, actions.RedisDeploymentName, actions.RBACName, labels) if err = controllerutil.SetControllerReference(instance, dp, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for Deployment: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for Deployment: %w", err)) } if updated, err = i.Ensure(ctx, dp); err != nil { @@ -50,13 +53,7 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Rekor) Reason: constants.Failure, Message: err.Error(), }) - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create Rekor redis: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create Rekor redis: %w", err), instance) } if updated { @@ -72,3 +69,26 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Rekor) } } + +func (i deployAction) CanHandleError(ctx context.Context, instance *rhtasv1alpha1.Rekor) bool { + err := i.Client.Get(ctx, types.NamespacedName{Name: actions.RedisDeploymentName, Namespace: instance.Namespace}, &v1.Deployment{}) + return !meta.IsStatusConditionTrue(instance.GetConditions(), actions.RedisCondition) && (err == nil || !errors.IsNotFound(err)) +} + +func (i deployAction) HandleError(ctx context.Context, instance *rhtasv1alpha1.Rekor) *action.Result { + redisDeployment := &v1.Deployment{} + if err := i.Client.Get(ctx, types.NamespacedName{Name: actions.RedisDeploymentName, Namespace: instance.Namespace}, redisDeployment); err != nil { + return i.Error(err) + } + if err := i.Client.Delete(ctx, redisDeployment); err != nil { + i.Logger.V(1).Info("Can't delete Redis deployment", "error", err.Error()) + } + + meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ + Type: actions.RedisCondition, + Status: metav1.ConditionFalse, + Reason: constants.Recovering, + Message: "Redis deployment will be recreated", + }) + return i.StatusUpdate(ctx, instance) +} diff --git a/internal/controller/rekor/actions/redis/initialize.go b/internal/controller/rekor/actions/redis/initialize.go index 8851547c6..85d9013a8 100644 --- a/internal/controller/rekor/actions/redis/initialize.go +++ b/internal/controller/rekor/actions/redis/initialize.go @@ -36,7 +36,7 @@ func (i initializeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Re labels := constants.LabelsForComponent(actions.RedisComponentName, instance.Name) ok, err = commonUtils.DeploymentIsRunning(ctx, i.Client, instance.Namespace, labels) if err != nil { - return i.Failed(err) + return i.Error(err) } if !ok { i.Logger.Info("Waiting for deployment") @@ -54,3 +54,11 @@ func (i initializeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Re return i.StatusUpdate(ctx, instance) } + +func (i initializeAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Rekor) bool { + return false +} + +func (i initializeAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Rekor) *action.Result { + return i.Continue() +} diff --git a/internal/controller/rekor/actions/redis/svc.go b/internal/controller/rekor/actions/redis/svc.go index bc634d614..ec0c56370 100644 --- a/internal/controller/rekor/actions/redis/svc.go +++ b/internal/controller/rekor/actions/redis/svc.go @@ -8,8 +8,11 @@ import ( k8sutils "github.com/securesign/operator/internal/controller/common/utils/kubernetes" "github.com/securesign/operator/internal/controller/constants" "github.com/securesign/operator/internal/controller/rekor/actions" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1" @@ -43,7 +46,7 @@ func (i createServiceAction) Handle(ctx context.Context, instance *rhtasv1alpha1 svc := k8sutils.CreateService(instance.Namespace, actions.RedisDeploymentName, actions.RedisDeploymentPortName, actions.RedisDeploymentPort, actions.RedisDeploymentPort, labels) if err = controllerutil.SetControllerReference(instance, svc, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for Redis service: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for Redis service: %w", err)) } if updated, err = i.Ensure(ctx, svc); err != nil { @@ -53,13 +56,7 @@ func (i createServiceAction) Handle(ctx context.Context, instance *rhtasv1alpha1 Reason: constants.Failure, Message: err.Error(), }) - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create service: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create service: %w", err), instance) } if updated { @@ -74,3 +71,26 @@ func (i createServiceAction) Handle(ctx context.Context, instance *rhtasv1alpha1 return i.Continue() } } + +func (i createServiceAction) CanHandleError(ctx context.Context, instance *rhtasv1alpha1.Rekor) bool { + err := i.Client.Get(ctx, types.NamespacedName{Name: actions.RedisDeploymentName, Namespace: instance.Namespace}, &v1.Service{}) + return !meta.IsStatusConditionTrue(instance.GetConditions(), actions.RedisCondition) && (err == nil || !errors.IsNotFound(err)) +} + +func (i createServiceAction) HandleError(ctx context.Context, instance *rhtasv1alpha1.Rekor) *action.Result { + svc := &v1.Service{} + if err := i.Client.Get(ctx, types.NamespacedName{Name: actions.RedisDeploymentName, Namespace: instance.Namespace}, svc); err != nil { + return i.Error(err) + } + if err := i.Client.Delete(ctx, svc); err != nil { + i.Logger.V(1).Info("Can't delete Redis service", "error", err.Error()) + } + + meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ + Type: actions.RedisCondition, + Status: metav1.ConditionFalse, + Reason: constants.Recovering, + Message: "Redis service will be recreated", + }) + return i.StatusUpdate(ctx, instance) +} diff --git a/internal/controller/rekor/actions/server/deployment.go b/internal/controller/rekor/actions/server/deployment.go index 0bc23c2f0..88ea15966 100644 --- a/internal/controller/rekor/actions/server/deployment.go +++ b/internal/controller/rekor/actions/server/deployment.go @@ -3,6 +3,7 @@ package server import ( "context" "fmt" + cutils "github.com/securesign/operator/internal/controller/common/utils" v1 "k8s.io/api/core/v1" @@ -11,8 +12,11 @@ import ( "github.com/securesign/operator/internal/controller/rekor/actions" "github.com/securesign/operator/internal/controller/rekor/utils" actions2 "github.com/securesign/operator/internal/controller/trillian/actions" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1" @@ -32,6 +36,9 @@ func (i deployAction) Name() string { func (i deployAction) CanHandle(_ context.Context, instance *rhtasv1alpha1.Rekor) bool { c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready) + if c == nil { + return false + } return c.Reason == constants.Creating || c.Reason == constants.Ready } @@ -59,16 +66,10 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Rekor) Reason: constants.Failure, Message: err.Error(), }) - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could create server Deployment: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could create server Deployment: %w", err), instance) } if err = controllerutil.SetControllerReference(instance, dp, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for Deployment: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for Deployment: %w", err)) } if updated, err = i.Ensure(ctx, dp); err != nil { @@ -78,13 +79,7 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Rekor) Reason: constants.Failure, Message: err.Error(), }) - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create Rekor server: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create Rekor server: %w", err), instance) } if updated { @@ -102,5 +97,28 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Rekor) } else { return i.Continue() } +} + +func (i deployAction) CanHandleError(ctx context.Context, instance *rhtasv1alpha1.Rekor) bool { + err := i.Client.Get(ctx, types.NamespacedName{Name: actions.ServerDeploymentName, Namespace: instance.Namespace}, &appsv1.Deployment{}) + return !meta.IsStatusConditionTrue(instance.GetConditions(), actions.ServerCondition) && (err == nil || !errors.IsNotFound(err)) +} + +func (i deployAction) HandleError(ctx context.Context, instance *rhtasv1alpha1.Rekor) *action.Result { + deployment := &appsv1.Deployment{} + if err := i.Client.Get(ctx, types.NamespacedName{Name: actions.ServerDeploymentName, Namespace: instance.Namespace}, deployment); err != nil { + return i.Error(err) + } + + if err := i.Client.Delete(ctx, deployment); err != nil { + i.Logger.V(1).Info("Can't delete server deployment", "error", err.Error()) + } + meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ + Type: actions.ServerCondition, + Status: metav1.ConditionFalse, + Reason: constants.Recovering, + Message: "server deployment will be recreated", + }) + return i.StatusUpdate(ctx, instance) } diff --git a/internal/controller/rekor/actions/server/generate_signer.go b/internal/controller/rekor/actions/server/generate_signer.go index 08efe546c..ddcda9918 100644 --- a/internal/controller/rekor/actions/server/generate_signer.go +++ b/internal/controller/rekor/actions/server/generate_signer.go @@ -9,7 +9,8 @@ import ( "crypto/x509" "encoding/pem" "fmt" - "github.com/securesign/operator/api/v1alpha1" + + rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1" "github.com/securesign/operator/internal/controller/common/action" k8sutils "github.com/securesign/operator/internal/controller/common/utils/kubernetes" "github.com/securesign/operator/internal/controller/constants" @@ -22,7 +23,7 @@ import ( const secretNameFormat = "rekor-signer-%s-" -func NewGenerateSignerAction() action.Action[*v1alpha1.Rekor] { +func NewGenerateSignerAction() action.Action[*rhtasv1alpha1.Rekor] { return &generateSigner{} } @@ -34,7 +35,7 @@ func (g generateSigner) Name() string { return "generate-signer" } -func (g generateSigner) CanHandle(_ context.Context, instance *v1alpha1.Rekor) bool { +func (g generateSigner) CanHandle(_ context.Context, instance *rhtasv1alpha1.Rekor) bool { c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready) if c == nil { return false @@ -47,17 +48,11 @@ func (g generateSigner) CanHandle(_ context.Context, instance *v1alpha1.Rekor) b } -func (g generateSigner) Handle(ctx context.Context, instance *v1alpha1.Rekor) *action.Result { +func (g generateSigner) Handle(ctx context.Context, instance *rhtasv1alpha1.Rekor) *action.Result { if instance.Spec.Signer.KMS != "secret" && instance.Spec.Signer.KMS != "" { instance.Status.Signer = instance.Spec.Signer // force recreation of public key ref instance.Status.PublicKeyRef = nil - // skip signer resolution and move to creating - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Creating, - }) meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ Type: actions.SignerCondition, Status: metav1.ConditionTrue, @@ -98,12 +93,6 @@ func (g generateSigner) Handle(ctx context.Context, instance *v1alpha1.Rekor) *a Reason: constants.Pending, Message: "resolving keys", }) - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Pending, - Message: "resolving keys", - }) return g.StatusUpdate(ctx, instance) } // swallow error and retry @@ -124,6 +113,7 @@ func (g generateSigner) Handle(ctx context.Context, instance *v1alpha1.Rekor) *a } secret := k8sutils.CreateImmutableSecret(fmt.Sprintf(secretNameFormat, instance.Name), instance.Namespace, data, labels) + if _, err = g.Ensure(ctx, secret); err != nil { meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ Type: actions.ServerCondition, @@ -131,29 +121,23 @@ func (g generateSigner) Handle(ctx context.Context, instance *v1alpha1.Rekor) *a Reason: constants.Failure, Message: err.Error(), }) - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return g.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create secret: %w", err), instance) + return g.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create secret: %w", err), instance) } g.Recorder.Eventf(instance, v1.EventTypeNormal, "SignerKeyCreated", "Signer private key created: %s", secret.Name) instance.Status.Signer = instance.Spec.Signer if instance.Spec.Signer.KeyRef == nil { - instance.Status.Signer.KeyRef = &v1alpha1.SecretKeySelector{ + instance.Status.Signer.KeyRef = &rhtasv1alpha1.SecretKeySelector{ Key: "private", - LocalObjectReference: v1alpha1.LocalObjectReference{ + LocalObjectReference: rhtasv1alpha1.LocalObjectReference{ Name: secret.Name, }, } } if _, ok := secret.Data["password"]; instance.Spec.Signer.PasswordRef == nil && ok { - instance.Status.Signer.PasswordRef = &v1alpha1.SecretKeySelector{ + instance.Status.Signer.PasswordRef = &rhtasv1alpha1.SecretKeySelector{ Key: "password", - LocalObjectReference: v1alpha1.LocalObjectReference{ + LocalObjectReference: rhtasv1alpha1.LocalObjectReference{ Name: secret.Name, }, } @@ -179,7 +163,7 @@ type RekorCertConfig struct { RekorKeyPassword []byte } -func (g generateSigner) CreateRekorKey(instance *v1alpha1.Rekor) (*RekorCertConfig, error) { +func (g generateSigner) CreateRekorKey(instance *rhtasv1alpha1.Rekor) (*RekorCertConfig, error) { var err error if instance.Spec.Signer.KeyRef != nil { config := &RekorCertConfig{} @@ -235,3 +219,21 @@ func (g generateSigner) CreateRekorKey(instance *v1alpha1.Rekor) (*RekorCertConf RekorPubKey: pemPubKey.Bytes(), }, nil } + +func (g generateSigner) CanHandleError(_ context.Context, instance *rhtasv1alpha1.Rekor) bool { + return !meta.IsStatusConditionTrue(instance.GetConditions(), actions.SignerCondition) && + (instance.Status.Signer.KeyRef != nil || instance.Status.Signer.PasswordRef != nil) +} + +func (g generateSigner) HandleError(ctx context.Context, instance *rhtasv1alpha1.Rekor) *action.Result { + instance.Status.Signer.PasswordRef = nil + instance.Status.Signer.KeyRef = nil + + meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ + Type: actions.SignerCondition, + Status: metav1.ConditionFalse, + Reason: constants.Recovering, + Message: "signer will be re-loaded", + }) + return g.StatusUpdate(ctx, instance) +} diff --git a/internal/controller/rekor/actions/server/ingress.go b/internal/controller/rekor/actions/server/ingress.go index 4462c4350..ea72a57f6 100644 --- a/internal/controller/rekor/actions/server/ingress.go +++ b/internal/controller/rekor/actions/server/ingress.go @@ -10,6 +10,8 @@ import ( "github.com/securesign/operator/internal/controller/constants" "github.com/securesign/operator/internal/controller/rekor/actions" v1 "k8s.io/api/core/v1" + v12 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -30,6 +32,9 @@ func (i ingressAction) Name() string { func (i ingressAction) CanHandle(_ context.Context, instance *rhtasv1alpha1.Rekor) bool { c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready) + if c == nil { + return false + } return (c.Reason == constants.Creating || c.Reason == constants.Ready) && instance.Spec.ExternalAccess.Enabled } @@ -41,16 +46,16 @@ func (i ingressAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Rekor svc := &v1.Service{} if err := i.Client.Get(ctx, ok, svc); err != nil { - return i.Failed(fmt.Errorf("could not find service for ingress: %w", err)) + return i.Error(fmt.Errorf("could not find service for ingress: %w", err)) } ingress, err := kubernetes.CreateIngress(ctx, i.Client, *svc, instance.Spec.ExternalAccess, actions.ServerDeploymentPortName, labels) if err != nil { - return i.Failed(fmt.Errorf("could not create ingress object: %w", err)) + return i.Error(fmt.Errorf("could not create ingress object: %w", err)) } if err = controllerutil.SetControllerReference(instance, ingress, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for Ingress: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for Ingress: %w", err)) } if updated, err = i.Ensure(ctx, ingress); err != nil { @@ -60,13 +65,7 @@ func (i ingressAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Rekor Reason: constants.Failure, Message: err.Error(), }) - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return i.FailedWithStatusUpdate(ctx, err, instance) + return i.ErrorWithStatusUpdate(ctx, err, instance) } if updated { @@ -81,3 +80,30 @@ func (i ingressAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Rekor return i.Continue() } } + +func (i ingressAction) CanHandleError(ctx context.Context, instance *rhtasv1alpha1.Rekor) bool { + err := i.Client.Get(ctx, types.NamespacedName{Name: actions.ServerDeploymentName, Namespace: instance.Namespace}, &v12.Ingress{}) + return !meta.IsStatusConditionTrue(instance.GetConditions(), actions.ServerCondition) && instance.Spec.ExternalAccess.Enabled && (err == nil || !errors.IsNotFound(err)) +} + +func (i ingressAction) HandleError(ctx context.Context, instance *rhtasv1alpha1.Rekor) *action.Result { + deployment := &v12.Ingress{} + if err := i.Client.Get(ctx, types.NamespacedName{Name: actions.ServerDeploymentName, Namespace: instance.Namespace}, deployment); err != nil { + if errors.IsNotFound(err) { + { + return i.Continue() + } + } + } + if err := i.Client.Delete(ctx, deployment); err != nil { + i.Logger.V(1).Info("Can't delete server ingress", "error", err.Error()) + } + + meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ + Type: actions.ServerCondition, + Status: metav1.ConditionFalse, + Reason: constants.Recovering, + Message: "server ingress will be recreated", + }) + return i.StatusUpdate(ctx, instance) +} diff --git a/internal/controller/rekor/actions/server/initialize.go b/internal/controller/rekor/actions/server/initialize.go index dc3ca5814..72d473f2a 100644 --- a/internal/controller/rekor/actions/server/initialize.go +++ b/internal/controller/rekor/actions/server/initialize.go @@ -38,7 +38,7 @@ func (i initializeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Re labels := constants.LabelsForComponent(actions.ServerComponentName, instance.Name) ok, err = commonUtils.DeploymentIsRunning(ctx, i.Client, instance.Namespace, labels) if err != nil { - return i.Failed(err) + return i.Error(err) } if !ok { i.Logger.Info("Waiting for deployment") @@ -50,7 +50,6 @@ func (i initializeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Re }) return i.StatusUpdate(ctx, instance) } - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ Type: actions.ServerCondition, Status: metav1.ConditionTrue, @@ -58,3 +57,11 @@ func (i initializeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Re }) return i.StatusUpdate(ctx, instance) } + +func (i initializeAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Rekor) bool { + return false +} + +func (i initializeAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Rekor) *action.Result { + return i.Continue() +} diff --git a/internal/controller/rekor/actions/server/monitoring.go b/internal/controller/rekor/actions/server/monitoring.go index 33731ef0c..d041427e0 100644 --- a/internal/controller/rekor/actions/server/monitoring.go +++ b/internal/controller/rekor/actions/server/monitoring.go @@ -11,8 +11,10 @@ import ( "github.com/securesign/operator/internal/controller/constants" "github.com/securesign/operator/internal/controller/rekor/actions" v1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -30,6 +32,9 @@ func (i monitoringAction) Name() string { func (i monitoringAction) CanHandle(_ context.Context, instance *rhtasv1alpha1.Rekor) bool { c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready) + if c == nil { + return false + } return (c.Reason == constants.Creating || c.Reason == constants.Ready) && instance.Spec.Monitoring.Enabled } @@ -54,7 +59,7 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Re ) if err = controllerutil.SetControllerReference(instance, role, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for role: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for role: %w", err)) } if _, err = i.Ensure(ctx, role); err != nil { @@ -64,13 +69,7 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Re Reason: constants.Failure, Message: err.Error(), }) - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create monitoring role: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create monitoring role: %w", err), instance) } roleBinding := kubernetes.CreateRoleBinding( @@ -87,7 +86,7 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Re }, ) if err = controllerutil.SetControllerReference(instance, roleBinding, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for role: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for role: %w", err)) } if _, err = i.Ensure(ctx, roleBinding); err != nil { @@ -97,13 +96,7 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Re Reason: constants.Failure, Message: err.Error(), }) - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create monitoring RoleBinding: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create monitoring RoleBinding: %w", err), instance) } serviceMonitor := kubernetes.CreateServiceMonitor( @@ -121,7 +114,7 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Re ) if err = controllerutil.SetControllerReference(instance, serviceMonitor, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for serviceMonitor: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for serviceMonitor: %w", err)) } if _, err = i.Ensure(ctx, serviceMonitor); err != nil { @@ -131,15 +124,32 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Re Reason: constants.Failure, Message: err.Error(), }) - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create serviceMonitor: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create serviceMonitor: %w", err), instance) } // monitors & RBAC are not watched - do not need to re-enqueue return i.Continue() } + +func (i monitoringAction) CanHandleError(ctx context.Context, instance *rhtasv1alpha1.Rekor) bool { + err := i.Client.Get(ctx, types.NamespacedName{Name: actions.ServerDeploymentName, Namespace: instance.Namespace}, &monitoringv1.ServiceMonitor{}) + return !meta.IsStatusConditionTrue(instance.GetConditions(), actions.ServerCondition) && instance.Spec.Monitoring.Enabled && (err == nil || !errors.IsNotFound(err)) +} + +func (i monitoringAction) HandleError(ctx context.Context, instance *rhtasv1alpha1.Rekor) *action.Result { + deployment := &monitoringv1.ServiceMonitor{} + if err := i.Client.Get(ctx, types.NamespacedName{Name: actions.ServerDeploymentName, Namespace: instance.Namespace}, deployment); err != nil { + return i.Error(err) + } + if err := i.Client.Delete(ctx, deployment); err != nil { + i.Logger.V(1).Info("Can't delete server deployment", "error", err.Error()) + } + + meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ + Type: actions.ServerCondition, + Status: metav1.ConditionFalse, + Reason: constants.Recovering, + Message: "servicemonitor will be recreated", + }) + return i.StatusUpdate(ctx, instance) +} diff --git a/internal/controller/rekor/actions/server/pvc.go b/internal/controller/rekor/actions/server/pvc.go index 80cd4f39f..f7272b2d6 100644 --- a/internal/controller/rekor/actions/server/pvc.go +++ b/internal/controller/rekor/actions/server/pvc.go @@ -34,6 +34,9 @@ func (i createPvcAction) Name() string { func (i createPvcAction) CanHandle(_ context.Context, instance *rhtasv1alpha1.Rekor) bool { c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready) + if c == nil { + return false + } return c.Reason == constants.Creating && instance.Status.PvcName == "" } @@ -45,7 +48,7 @@ func (i createPvcAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Rek } if instance.Spec.Pvc.Size == nil { - return i.Failed(fmt.Errorf("PVC size is not set")) + return i.Error(fmt.Errorf("PVC size is not set")) } // PVC does not exist, create a new one @@ -53,7 +56,7 @@ func (i createPvcAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Rek pvc := k8sutils.CreatePVC(instance.Namespace, fmt.Sprintf(PvcNameFormat, instance.Name), *instance.Spec.Pvc.Size, instance.Spec.Pvc.StorageClass, constants.LabelsFor(actions.ServerComponentName, actions.ServerDeploymentName, instance.Name)) if !utils.OptionalBool(instance.Spec.Pvc.Retain) { if err = controllerutil.SetControllerReference(instance, pvc, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for PVC: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for PVC: %w", err)) } } @@ -64,15 +67,18 @@ func (i createPvcAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Rek Reason: constants.Failure, Message: err.Error(), }) - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create DB PVC: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create DB PVC: %w", err), instance) } i.Recorder.Event(instance, v1.EventTypeNormal, "PersistentVolumeCreated", "New PersistentVolume created") instance.Status.PvcName = pvc.Name return i.StatusUpdate(ctx, instance) } + +func (i createPvcAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Rekor) bool { + // do not delete any PCV + return false +} + +func (i createPvcAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Rekor) *action.Result { + return i.Continue() +} diff --git a/internal/controller/rekor/actions/server/resolve_pub_key.go b/internal/controller/rekor/actions/server/resolve_pub_key.go index ee72fe633..873704e29 100644 --- a/internal/controller/rekor/actions/server/resolve_pub_key.go +++ b/internal/controller/rekor/actions/server/resolve_pub_key.go @@ -31,6 +31,8 @@ const ( pubSecretNameFormat = "rekor-public-%s-" ) +var HttpClient = &http.Client{Timeout: 10 * time.Second} + func NewResolvePubKeyAction() action.Action[*rhtasv1alpha1.Rekor] { return &resolvePubKeyAction{} } @@ -58,13 +60,7 @@ func (i resolvePubKeyAction) Handle(ctx context.Context, instance *rhtasv1alpha1 publicKey, err = i.resolvePubKey(*instance) if err != nil { errf := fmt.Errorf("ResolvePubKey: unable to resolve public key: %v", err) - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: actions.ServerCondition, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: errf.Error(), - }) - return i.FailedWithStatusUpdate(ctx, errf, instance) + return i.ErrorWithStatusUpdate(ctx, errf, instance) } scrl := &metav1.PartialObjectMetadataList{} @@ -75,7 +71,7 @@ func (i resolvePubKeyAction) Handle(ctx context.Context, instance *rhtasv1alpha1 }) if err = k8sutils.FindByLabelSelector(ctx, i.Client, scrl, instance.Namespace, RekorPubLabel); err != nil { - return i.Failed(fmt.Errorf("ResolvePubKey: find secrets failed: %w", err)) + return i.Error(fmt.Errorf("ResolvePubKey: find secrets failed: %w", err)) } // Search if exists a secret with rhtas.redhat.com/rekor.pub label @@ -89,7 +85,7 @@ func (i resolvePubKeyAction) Handle(ctx context.Context, instance *rhtasv1alpha1 var sksPublicKey []byte sksPublicKey, err = k8sutils.GetSecretData(i.Client, instance.Namespace, &sks) if err != nil { - return i.Failed(fmt.Errorf("ResolvePubKey: failed to read `%s` secret's data: %w", sks.Name, err)) + return i.Error(fmt.Errorf("ResolvePubKey: failed to read `%s` secret's data: %w", sks.Name, err)) } if bytes.Equal(sksPublicKey, publicKey) { @@ -99,7 +95,7 @@ func (i resolvePubKeyAction) Handle(ctx context.Context, instance *rhtasv1alpha1 // Remove label from secret if err = i.removeLabel(ctx, &secret); err != nil { - return i.Failed(fmt.Errorf("ResolvePubKey: %w", err)) + return i.Error(fmt.Errorf("ResolvePubKey: %w", err)) } message := fmt.Sprintf("Removed '%s' label from %s secret", RekorPubLabel, secret.Name) @@ -130,13 +126,7 @@ func (i resolvePubKeyAction) Handle(ctx context.Context, instance *rhtasv1alpha1 newConfig.Annotations[annotations.TreeId] = strconv.FormatInt(pointer.Int64Deref(instance.Status.TreeID, 0), 10) if err = i.Client.Create(ctx, newConfig); err != nil { - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: actions.ServerCondition, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return i.FailedWithStatusUpdate(ctx, err, instance) + return i.ErrorWithStatusUpdate(ctx, err, instance) } i.Recorder.Eventf(instance, v1.EventTypeNormal, "PublicKeySecretCreated", "New Rekor public key created: %s", newConfig.Name) @@ -155,14 +145,13 @@ func (i resolvePubKeyAction) resolvePubKey(instance rhtasv1alpha1.Rekor) ([]byte if data, err = i.requestPublicKey(fmt.Sprintf("http://%s.%s.svc", actions.ServerDeploymentName, instance.Namespace)); err == nil { return data, nil } - i.Logger.Info("retrying to get rekor public key") return nil, err } func (i resolvePubKeyAction) requestPublicKey(basePath string) ([]byte, error) { - c := http.Client{Timeout: 1 * time.Minute} - response, err := c.Get(fmt.Sprintf("%s/api/v1/log/publicKey", basePath)) + + response, err := HttpClient.Get(fmt.Sprintf("%s/api/v1/log/publicKey", basePath)) if err != nil { return nil, err } @@ -202,3 +191,12 @@ func (i resolvePubKeyAction) removeLabel(ctx context.Context, object *metav1.Par return nil } + +func (i resolvePubKeyAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Rekor) bool { + // all leftovers are removed in generate_signer action + return false +} + +func (i resolvePubKeyAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Rekor) *action.Result { + return i.Continue() +} diff --git a/internal/controller/rekor/actions/server/resolve_pub_key_test.go b/internal/controller/rekor/actions/server/resolve_pub_key_test.go index e48dbad0f..849ce7f4e 100644 --- a/internal/controller/rekor/actions/server/resolve_pub_key_test.go +++ b/internal/controller/rekor/actions/server/resolve_pub_key_test.go @@ -4,19 +4,20 @@ import ( "bytes" "context" "fmt" + "io" + "net/http" + "reflect" + "testing" + . "github.com/onsi/gomega" "github.com/securesign/operator/internal/controller/common/action" "github.com/securesign/operator/internal/controller/common/utils/kubernetes" "github.com/securesign/operator/internal/controller/rekor/actions" testAction "github.com/securesign/operator/internal/testing/action" httpmock "github.com/securesign/operator/internal/testing/http" - "io" v1 "k8s.io/api/core/v1" "k8s.io/utils/pointer" - "net/http" - "reflect" "sigs.k8s.io/controller-runtime/pkg/client" - "testing" "github.com/securesign/operator/api/v1alpha1" "github.com/securesign/operator/internal/controller/constants" @@ -142,7 +143,7 @@ func TestResolvePubKey_Handle(t *testing.T) { { name: "unable to resolve public key", want: want{ - result: testAction.FailedWithStatusUpdate(fmt.Errorf("ResolvePubKey: unable to resolve public key: unexpected http response ")), + result: testAction.ErrorWithStatusUpdate(fmt.Errorf("ResolvePubKey: unable to resolve public key: unexpected http response ")), publicKey: nil, }, }, @@ -172,7 +173,7 @@ func TestResolvePubKey_Handle(t *testing.T) { WithObjects(instance). WithStatusSubresource(instance). WithObjects(tt.env.objects...).Build() - httpmock.SetMockTransport(http.DefaultClient, map[string]httpmock.RoundTripFunc{ + httpmock.SetMockTransport(HttpClient, map[string]httpmock.RoundTripFunc{ "http://rekor-server.default.svc/api/v1/log/publicKey": func(req *http.Request) *http.Response { if tt.want.publicKey == nil { return &http.Response{ @@ -187,7 +188,7 @@ func TestResolvePubKey_Handle(t *testing.T) { } }, }) - defer httpmock.RestoreDefaultTransport(http.DefaultClient) + defer httpmock.RestoreDefaultTransport(HttpClient) a := testAction.PrepareAction(c, NewResolvePubKeyAction()) diff --git a/internal/controller/rekor/actions/server/resolve_tree.go b/internal/controller/rekor/actions/server/resolve_tree.go index 7cfd1435c..d8afda83c 100644 --- a/internal/controller/rekor/actions/server/resolve_tree.go +++ b/internal/controller/rekor/actions/server/resolve_tree.go @@ -73,7 +73,7 @@ func (i resolveTreeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.R trillUrl = fmt.Sprintf("%s:%d", instance.Spec.Trillian.Address, *instance.Spec.Trillian.Port) } if err != nil { - return i.Failed(err) + return i.Error(err) } i.Logger.V(1).Info("trillian logserver", "address", trillUrl) @@ -91,10 +91,20 @@ func (i resolveTreeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.R Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create trillian tree: %v", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create trillian tree: %v", err), instance) } i.Recorder.Eventf(instance, v1.EventTypeNormal, "TrillianTreeCreated", "New Trillian tree created: %d", tree.TreeId) instance.Status.TreeID = &tree.TreeId return i.StatusUpdate(ctx, instance) } + +func (i resolveTreeAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Rekor) bool { + // instance.Status.TreeID == nil in case of failure + // no need to recover + return false +} + +func (i resolveTreeAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Rekor) *action.Result { + return i.Continue() +} diff --git a/internal/controller/rekor/actions/server/resolve_tree_test.go b/internal/controller/rekor/actions/server/resolve_tree_test.go index bdf9d1882..15fce0446 100644 --- a/internal/controller/rekor/actions/server/resolve_tree_test.go +++ b/internal/controller/rekor/actions/server/resolve_tree_test.go @@ -4,6 +4,9 @@ import ( "context" "errors" "fmt" + "reflect" + "testing" + "github.com/google/trillian" . "github.com/onsi/gomega" rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1" @@ -15,8 +18,6 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" - "reflect" - "testing" ) func TestResolveTree_CanHandle(t *testing.T) { @@ -192,7 +193,7 @@ func TestResolveTree_Handle(t *testing.T) { createTree: mockCreateTree(nil, errors.New("timeout error"), nil), }, want: want{ - result: testAction.FailedWithStatusUpdate(fmt.Errorf("could not create trillian tree: timeout error")), + result: testAction.ErrorWithStatusUpdate(fmt.Errorf("could not create trillian tree: timeout error")), verify: func(g Gomega, rekor *rhtasv1alpha1.Rekor) { g.Expect(rekor.Spec.TreeID).Should(BeNil()) g.Expect(rekor.Status.TreeID).Should(BeNil()) diff --git a/internal/controller/rekor/actions/server/servrConfig.go b/internal/controller/rekor/actions/server/servrConfig.go index d379ae4b3..58ad3db78 100644 --- a/internal/controller/rekor/actions/server/servrConfig.go +++ b/internal/controller/rekor/actions/server/servrConfig.go @@ -9,6 +9,7 @@ import ( "github.com/securesign/operator/internal/controller/common/utils/kubernetes" "github.com/securesign/operator/internal/controller/constants" "github.com/securesign/operator/internal/controller/rekor/actions" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -32,6 +33,9 @@ func (i serverConfig) Name() string { func (i serverConfig) CanHandle(_ context.Context, instance *rhtasv1alpha1.Rekor) bool { c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready) + if c == nil { + return false + } return c.Reason == constants.Creating && instance.Status.ServerConfigRef == nil } @@ -42,11 +46,11 @@ func (i serverConfig) Handle(ctx context.Context, instance *rhtasv1alpha1.Rekor) labels := constants.LabelsFor(actions.ServerComponentName, actions.ServerDeploymentName, instance.Name) if err != nil { - return i.FailedWithStatusUpdate(ctx, err, instance) + return i.ErrorWithStatusUpdate(ctx, err, instance) } newConfig := kubernetes.CreateImmutableConfigmap(fmt.Sprintf("rekor-server-config-%s", instance.Namespace), instance.Namespace, labels, map[string]string{"sharding-config.yaml": ""}) if err = controllerutil.SetControllerReference(instance, newConfig, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for ConfigMap: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for ConfigMap: %w", err)) } _, err = i.Ensure(ctx, newConfig) @@ -57,13 +61,7 @@ func (i serverConfig) Handle(ctx context.Context, instance *rhtasv1alpha1.Rekor) Reason: constants.Failure, Message: err.Error(), }) - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return i.FailedWithStatusUpdate(ctx, err, instance) + return i.ErrorWithStatusUpdate(ctx, err, instance) } instance.Status.ServerConfigRef = &rhtasv1alpha1.LocalObjectReference{Name: newConfig.Name} @@ -76,3 +74,29 @@ func (i serverConfig) Handle(ctx context.Context, instance *rhtasv1alpha1.Rekor) }) return i.StatusUpdate(ctx, instance) } + +func (i serverConfig) CanHandleError(_ context.Context, instance *rhtasv1alpha1.Rekor) bool { + return !meta.IsStatusConditionTrue(instance.GetConditions(), actions.ServerCondition) && instance.Status.ServerConfigRef != nil +} + +func (i serverConfig) HandleError(ctx context.Context, instance *rhtasv1alpha1.Rekor) *action.Result { + deployment := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Status.ServerConfigRef.Name, + Namespace: instance.Namespace, + }, + } + if err := i.Client.Delete(ctx, deployment); err != nil { + i.Logger.V(1).Info("Can't delete server configuration", "error", err.Error()) + } + + instance.Status.ServerConfigRef = nil + + meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ + Type: actions.ServerCondition, + Status: metav1.ConditionFalse, + Reason: constants.Recovering, + Message: "server configuration will be recreated", + }) + return i.StatusUpdate(ctx, instance) +} diff --git a/internal/controller/rekor/actions/server/status_url.go b/internal/controller/rekor/actions/server/status_url.go index ba9ea787d..4936430fa 100644 --- a/internal/controller/rekor/actions/server/status_url.go +++ b/internal/controller/rekor/actions/server/status_url.go @@ -28,6 +28,9 @@ func (i statusUrlAction) Name() string { func (i statusUrlAction) CanHandle(_ context.Context, instance *rhtasv1alpha1.Rekor) bool { c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready) + if c == nil { + return false + } return c.Reason == constants.Creating || c.Reason == constants.Ready } @@ -38,7 +41,7 @@ func (i statusUrlAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Rek ingress := &v12.Ingress{} err := i.Client.Get(ctx, types.NamespacedName{Name: actions.ServerDeploymentName, Namespace: instance.Namespace}, ingress) if err != nil { - return i.Failed(err) + return i.Error(err) } if len(ingress.Spec.TLS) > 0 { protocol = "https://" @@ -55,3 +58,11 @@ func (i statusUrlAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Rek instance.Status.Url = url return i.StatusUpdate(ctx, instance) } + +func (i statusUrlAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Rekor) bool { + return false +} + +func (i statusUrlAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Rekor) *action.Result { + return i.Continue() +} diff --git a/internal/controller/rekor/actions/server/svc.go b/internal/controller/rekor/actions/server/svc.go index eb9def35e..013768377 100644 --- a/internal/controller/rekor/actions/server/svc.go +++ b/internal/controller/rekor/actions/server/svc.go @@ -9,8 +9,10 @@ import ( "github.com/securesign/operator/internal/controller/constants" "github.com/securesign/operator/internal/controller/rekor/actions" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -31,6 +33,9 @@ func (i createServiceAction) Name() string { func (i createServiceAction) CanHandle(_ context.Context, instance *rhtasv1alpha1.Rekor) bool { c := meta.FindStatusCondition(instance.Status.Conditions, constants.Ready) + if c == nil { + return false + } return c.Reason == constants.Creating || c.Reason == constants.Ready } @@ -54,7 +59,7 @@ func (i createServiceAction) Handle(ctx context.Context, instance *rhtasv1alpha1 } if err = controllerutil.SetControllerReference(instance, svc, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for service: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for service: %w", err)) } if updated, err = i.Ensure(ctx, svc); err != nil { @@ -64,13 +69,7 @@ func (i createServiceAction) Handle(ctx context.Context, instance *rhtasv1alpha1 Reason: constants.Failure, Message: err.Error(), }) - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create service: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create service: %w", err), instance) } if updated { @@ -85,3 +84,26 @@ func (i createServiceAction) Handle(ctx context.Context, instance *rhtasv1alpha1 return i.Continue() } } + +func (i createServiceAction) CanHandleError(ctx context.Context, instance *rhtasv1alpha1.Rekor) bool { + err := i.Client.Get(ctx, types.NamespacedName{Name: actions.ServerDeploymentName, Namespace: instance.Namespace}, &corev1.Service{}) + return !meta.IsStatusConditionTrue(instance.GetConditions(), actions.ServerCondition) && (err == nil || !errors.IsNotFound(err)) +} + +func (i createServiceAction) HandleError(ctx context.Context, instance *rhtasv1alpha1.Rekor) *action.Result { + deployment := &corev1.Service{} + if err := i.Client.Get(ctx, types.NamespacedName{Name: actions.ServerDeploymentName, Namespace: instance.Namespace}, deployment); err != nil { + return i.Error(err) + } + if err := i.Client.Delete(ctx, deployment); err != nil { + i.Logger.V(1).Info("Can't delete service", "error", err.Error()) + } + + meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ + Type: actions.ServerCondition, + Status: metav1.ConditionFalse, + Reason: constants.Recovering, + Message: "server service will be recreated", + }) + return i.StatusUpdate(ctx, instance) +} diff --git a/internal/controller/rekor/actions/ui/deployment.go b/internal/controller/rekor/actions/ui/deployment.go index 8c7301e4d..5bac6344e 100644 --- a/internal/controller/rekor/actions/ui/deployment.go +++ b/internal/controller/rekor/actions/ui/deployment.go @@ -9,8 +9,11 @@ import ( "github.com/securesign/operator/internal/controller/constants" "github.com/securesign/operator/internal/controller/rekor/actions" "github.com/securesign/operator/internal/controller/rekor/utils" + v1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1" @@ -44,7 +47,7 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Rekor) labels := constants.LabelsFor(actions.UIComponentName, actions.SearchUiDeploymentName, instance.Name) dp := utils.CreateRekorSearchUiDeployment(instance, actions.SearchUiDeploymentName, actions.RBACName, labels) if err = controllerutil.SetControllerReference(instance, dp, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for Deployment: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for Deployment: %w", err)) } if updated, err = i.Ensure(ctx, dp); err != nil { @@ -54,13 +57,7 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Rekor) Reason: constants.Failure, Message: err.Error(), }) - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create Rekor search UI: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create Rekor search UI: %w", err), instance) } if updated { @@ -74,5 +71,27 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Rekor) } else { return i.Continue() } +} + +func (i deployAction) CanHandleError(ctx context.Context, instance *rhtasv1alpha1.Rekor) bool { + err := i.Client.Get(ctx, types.NamespacedName{Name: actions.SearchUiDeploymentName, Namespace: instance.Namespace}, &v1.Deployment{}) + return commonutils.IsEnabled(instance.Spec.RekorSearchUI.Enabled) && !meta.IsStatusConditionTrue(instance.GetConditions(), actions.UICondition) && (err == nil || !errors.IsNotFound(err)) +} + +func (i deployAction) HandleError(ctx context.Context, instance *rhtasv1alpha1.Rekor) *action.Result { + deployment := &v1.Deployment{} + if err := i.Client.Get(ctx, types.NamespacedName{Name: actions.SearchUiDeploymentName, Namespace: instance.Namespace}, deployment); err != nil { + return i.Error(err) + } + if err := i.Client.Delete(ctx, deployment); err != nil { + i.Logger.V(1).Info("Can't delete UI deployment", "error", err.Error()) + } + meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ + Type: actions.UICondition, + Status: metav1.ConditionFalse, + Reason: constants.Recovering, + Message: "UI deployment will be recreated", + }) + return i.StatusUpdate(ctx, instance) } diff --git a/internal/controller/rekor/actions/ui/ingress.go b/internal/controller/rekor/actions/ui/ingress.go index d6d18e166..0d64371c4 100644 --- a/internal/controller/rekor/actions/ui/ingress.go +++ b/internal/controller/rekor/actions/ui/ingress.go @@ -11,6 +11,8 @@ import ( "github.com/securesign/operator/internal/controller/constants" "github.com/securesign/operator/internal/controller/rekor/actions" v1 "k8s.io/api/core/v1" + v12 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -45,16 +47,16 @@ func (i ingressAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Rekor svc := &v1.Service{} if err := i.Client.Get(ctx, ok, svc); err != nil { - return i.Failed(fmt.Errorf("could not find service for ingress: %w", err)) + return i.Error(fmt.Errorf("could not find service for ingress: %w", err)) } ingress, err := kubernetes.CreateIngress(ctx, i.Client, *svc, rhtasv1alpha1.ExternalAccess{Host: instance.Spec.ExternalAccess.Host}, actions.SearchUiDeploymentPortName, labels) if err != nil { - return i.Failed(fmt.Errorf("could not create ingress object: %w", err)) + return i.Error(fmt.Errorf("could not create ingress object: %w", err)) } if err = controllerutil.SetControllerReference(instance, ingress, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for Ingress: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for Ingress: %w", err)) } if updated, err = i.Ensure(ctx, ingress); err != nil { @@ -70,7 +72,7 @@ func (i ingressAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Rekor Reason: constants.Failure, Message: err.Error(), }) - return i.Failed(fmt.Errorf("could not create Ingress: %w", err)) + return i.Error(fmt.Errorf("could not create Ingress: %w", err)) } if updated { @@ -85,3 +87,26 @@ func (i ingressAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Rekor return i.Continue() } } + +func (i ingressAction) CanHandleError(ctx context.Context, instance *rhtasv1alpha1.Rekor) bool { + err := i.Client.Get(ctx, types.NamespacedName{Name: actions.SearchUiDeploymentName, Namespace: instance.Namespace}, &v12.Ingress{}) + return utils.IsEnabled(instance.Spec.RekorSearchUI.Enabled) && !meta.IsStatusConditionTrue(instance.GetConditions(), actions.UICondition) && (err == nil || !errors.IsNotFound(err)) +} + +func (i ingressAction) HandleError(ctx context.Context, instance *rhtasv1alpha1.Rekor) *action.Result { + deployment := &v12.Ingress{} + if err := i.Client.Get(ctx, types.NamespacedName{Name: actions.SearchUiDeploymentName, Namespace: instance.Namespace}, deployment); err != nil { + return i.Error(err) + } + if err := i.Client.Delete(ctx, deployment); err != nil { + i.Logger.V(1).Info("Can't delete UI ingress", "error", err.Error()) + } + + meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ + Type: actions.UICondition, + Status: metav1.ConditionFalse, + Reason: constants.Recovering, + Message: "UI ingress will be recreated", + }) + return i.StatusUpdate(ctx, instance) +} diff --git a/internal/controller/rekor/actions/ui/initialize.go b/internal/controller/rekor/actions/ui/initialize.go index c46af3653..1ee21a484 100644 --- a/internal/controller/rekor/actions/ui/initialize.go +++ b/internal/controller/rekor/actions/ui/initialize.go @@ -41,7 +41,7 @@ func (i initializeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Re labels := constants.LabelsForComponent(actions.UIComponentName, instance.Name) ok, err = commonUtils.DeploymentIsRunning(ctx, i.Client, instance.Namespace, labels) if err != nil { - return i.Failed(err) + return i.Error(err) } if !ok { i.Logger.Info("Waiting for deployment") @@ -59,7 +59,7 @@ func (i initializeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Re err = i.Client.Get(ctx, types.NamespacedName{Name: actions.SearchUiDeploymentName, Namespace: instance.Namespace}, ingress) if err != nil { // condition error - return i.FailedWithStatusUpdate(ctx, err, instance) + return i.ErrorWithStatusUpdate(ctx, err, instance) } if len(ingress.Spec.TLS) > 0 { protocol = "https://" @@ -71,3 +71,11 @@ func (i initializeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Re return i.StatusUpdate(ctx, instance) } + +func (i initializeAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Rekor) bool { + return false +} + +func (i initializeAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Rekor) *action.Result { + return i.Continue() +} diff --git a/internal/controller/rekor/actions/ui/svc.go b/internal/controller/rekor/actions/ui/svc.go index aee32f5e8..78426947e 100644 --- a/internal/controller/rekor/actions/ui/svc.go +++ b/internal/controller/rekor/actions/ui/svc.go @@ -9,8 +9,11 @@ import ( k8sutils "github.com/securesign/operator/internal/controller/common/utils/kubernetes" "github.com/securesign/operator/internal/controller/constants" "github.com/securesign/operator/internal/controller/rekor/actions" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1" @@ -45,7 +48,7 @@ func (i createServiceAction) Handle(ctx context.Context, instance *rhtasv1alpha1 svc.Spec.Ports[0].Port = 80 if err = controllerutil.SetControllerReference(instance, svc, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for service: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for service: %w", err)) } if updated, err = i.Ensure(ctx, svc); err != nil { @@ -55,13 +58,7 @@ func (i createServiceAction) Handle(ctx context.Context, instance *rhtasv1alpha1 Reason: constants.Failure, Message: err.Error(), }) - meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ - Type: constants.Ready, - Status: metav1.ConditionFalse, - Reason: constants.Failure, - Message: err.Error(), - }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create service: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create service: %w", err), instance) } if updated { @@ -76,3 +73,26 @@ func (i createServiceAction) Handle(ctx context.Context, instance *rhtasv1alpha1 return i.Continue() } } + +func (i createServiceAction) CanHandleError(ctx context.Context, instance *rhtasv1alpha1.Rekor) bool { + err := i.Client.Get(ctx, types.NamespacedName{Name: actions.SearchUiDeploymentName, Namespace: instance.Namespace}, &v1.Service{}) + return utils.IsEnabled(instance.Spec.RekorSearchUI.Enabled) && !meta.IsStatusConditionTrue(instance.GetConditions(), actions.UICondition) && (err == nil || !errors.IsNotFound(err)) +} + +func (i createServiceAction) HandleError(ctx context.Context, instance *rhtasv1alpha1.Rekor) *action.Result { + deployment := &v1.Service{} + if err := i.Client.Get(ctx, types.NamespacedName{Name: actions.SearchUiDeploymentName, Namespace: instance.Namespace}, deployment); err != nil { + return i.Error(err) + } + if err := i.Client.Delete(ctx, deployment); err != nil { + i.Logger.V(1).Info("Can't delete UI service", "error", err.Error()) + } + + meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ + Type: actions.UICondition, + Status: metav1.ConditionFalse, + Reason: constants.Recovering, + Message: "UI service will be recreated", + }) + return i.StatusUpdate(ctx, instance) +} diff --git a/internal/controller/rekor/rekor_controller.go b/internal/controller/rekor/rekor_controller.go index db8954ae9..1b78700f5 100644 --- a/internal/controller/rekor/rekor_controller.go +++ b/internal/controller/rekor/rekor_controller.go @@ -18,11 +18,17 @@ package rekor import ( "context" + + "github.com/securesign/operator/internal/controller/common/utils" "k8s.io/apimachinery/pkg/types" olpredicate "github.com/operator-framework/operator-lib/predicate" + "github.com/securesign/operator/internal/apis" "github.com/securesign/operator/internal/controller/annotations" "github.com/securesign/operator/internal/controller/common/action/transitions" + tasPredicate "github.com/securesign/operator/internal/controller/common/predicate" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/predicate" actions2 "github.com/securesign/operator/internal/controller/rekor/actions" backfillredis "github.com/securesign/operator/internal/controller/rekor/actions/backfillRedis" @@ -75,7 +81,7 @@ type RekorReconciler struct { func (r *RekorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { var instance rhtasv1alpha1.Rekor log := ctrllog.FromContext(ctx) - log.V(1).Info("Reconciling Rekor", "request", req) + log.V(2).Info("Reconciling Rekor", "request", req) if err := r.Client.Get(ctx, req.NamespacedName, &instance); err != nil { return reconcile.Result{}, client.IgnoreNotFound(err) @@ -97,9 +103,12 @@ func (r *RekorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl actions := []action.Action[*rhtasv1alpha1.Rekor]{ transitions.NewToPendingPhaseAction[*rhtasv1alpha1.Rekor](func(rekor *rhtasv1alpha1.Rekor) []string { components := []string{actions2.ServerCondition, actions2.RedisCondition, actions2.SignerCondition} - if *rekor.Spec.RekorSearchUI.Enabled { + if utils.IsEnabled(rekor.Spec.RekorSearchUI.Enabled) { components = append(components, actions2.UICondition) } + if utils.IsEnabled(rekor.Spec.BackFillRedis.Enabled) { + components = append(components, actions2.BackfillRedisCondition) + } return components }), @@ -129,13 +138,17 @@ func (r *RekorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl transitions.NewToInitializePhaseAction[*rhtasv1alpha1.Rekor](), // INITIALIZE server.NewInitializeAction(), - server.NewResolvePubKeyAction(), ui.NewInitializeAction(), redis.NewInitializeAction(), + server.NewResolvePubKeyAction(), + // INITIALIZE -> READY actions2.NewInitializeAction(), + + // this should be always the last one + transitions.NewRestartOnErrorAction[*rhtasv1alpha1.Rekor](), } for _, a := range actions { @@ -143,11 +156,21 @@ func (r *RekorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl a.InjectLogger(log.WithName(a.Name())) a.InjectRecorder(r.Recorder) - if a.CanHandle(ctx, target) { - log.V(2).Info("Executing " + a.Name()) - result := a.Handle(ctx, target) - if result != nil { - return result.Result, result.Err + if apis.IsError(&instance) { + if a.CanHandleError(ctx, target) { + log.V(1).Info("Executing error handling action " + a.Name()) + result := a.HandleError(ctx, target) + if result != nil { + return result.Result, result.Err + } + } + } else { + if a.CanHandle(ctx, target) { + log.V(1).Info("Executing " + a.Name()) + result := a.Handle(ctx, target) + if result != nil { + return result.Result, result.Err + } } } } @@ -164,7 +187,10 @@ func (r *RekorReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). WithEventFilter(pause). - For(&rhtasv1alpha1.Rekor{}). + For(&rhtasv1alpha1.Rekor{}, builder.WithPredicates(predicate.And( + predicate.Or(predicate.GenerationChangedPredicate{}, tasPredicate.WaitOnError[*rhtasv1alpha1.Rekor]()), + tasPredicate.StopOnFailure[*rhtasv1alpha1.Rekor]()), + )). Owns(&v12.Deployment{}). Owns(&v13.Service{}). Owns(&v1.Ingress{}). diff --git a/internal/controller/rekor/rekor_controller_test.go b/internal/controller/rekor/rekor_controller_test.go index e5b704dcb..a9c2ab305 100644 --- a/internal/controller/rekor/rekor_controller_test.go +++ b/internal/controller/rekor/rekor_controller_test.go @@ -19,13 +19,14 @@ package rekor import ( "bytes" "context" - "github.com/securesign/operator/internal/controller/common/utils/kubernetes" - "github.com/securesign/operator/internal/controller/rekor/actions/server" - httpmock "github.com/securesign/operator/internal/testing/http" "io" "net/http" "time" + "github.com/securesign/operator/internal/controller/common/utils/kubernetes" + "github.com/securesign/operator/internal/controller/rekor/actions/server" + httpmock "github.com/securesign/operator/internal/testing/http" + "github.com/securesign/operator/internal/controller/common/utils" "github.com/securesign/operator/api/v1alpha1" @@ -49,15 +50,14 @@ var _ = Describe("Rekor controller", func() { const ( Name = "test" - Namespace = "default" + Namespace = "controller" ) ctx := context.Background() namespace := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: Name, - Namespace: Namespace, + Name: Namespace, }, } @@ -153,8 +153,8 @@ var _ = Describe("Rekor controller", func() { }) Expect(err).To(Succeed()) - httpmock.SetMockTransport(http.DefaultClient, map[string]httpmock.RoundTripFunc{ - "http://rekor-server.default.svc/api/v1/log/publicKey": func(req *http.Request) *http.Response { + httpmock.SetMockTransport(server.HttpClient, map[string]httpmock.RoundTripFunc{ + "http://rekor-server.controller.svc/api/v1/log/publicKey": func(req *http.Request) *http.Response { return &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewReader(pubKeyData)), @@ -162,6 +162,7 @@ var _ = Describe("Rekor controller", func() { } }, }) + defer httpmock.RestoreDefaultTransport(server.HttpClient) By("Rekor server PVC created") Eventually(func(g Gomega) string { diff --git a/internal/controller/rekor/rekor_error_handler_test.go b/internal/controller/rekor/rekor_error_handler_test.go index fd4fcc39c..d9c4049da 100644 --- a/internal/controller/rekor/rekor_error_handler_test.go +++ b/internal/controller/rekor/rekor_error_handler_test.go @@ -24,18 +24,18 @@ import ( "os" "time" + "github.com/securesign/operator/api/v1alpha1" "github.com/securesign/operator/internal/controller/common/utils" "github.com/securesign/operator/internal/controller/common/utils/kubernetes" + "github.com/securesign/operator/internal/controller/constants" + "github.com/securesign/operator/internal/controller/rekor/actions/server" trillian "github.com/securesign/operator/internal/controller/trillian/actions" httpmock "github.com/securesign/operator/internal/testing/http" appsv1 "k8s.io/api/apps/v1" - runtimeClient "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/securesign/operator/api/v1alpha1" - "github.com/securesign/operator/internal/controller/constants" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" + runtimeClient "sigs.k8s.io/controller-runtime/pkg/client" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -116,7 +116,7 @@ var _ = Describe("Rekor ErrorHandler", func() { err = k8sClient.Create(ctx, instance) Expect(err).To(Not(HaveOccurred())) } - Expect(k8sClient.Create(ctx, kubernetes.CreateService(Namespace, trillian.LogserverDeploymentName, trillian.ServerPortName, trillian.ServerPort, constants.LabelsForComponent(trillian.LogServerComponentName, instance.Name)))).To(Succeed()) + Expect(k8sClient.Create(ctx, kubernetes.CreateService(Namespace, trillian.LogserverDeploymentName, trillian.ServerPortName, trillian.ServerPort, trillian.ServerPort, constants.LabelsForComponent(trillian.LogServerComponentName, instance.Name)))).To(Succeed()) found := &v1alpha1.Rekor{} @@ -139,12 +139,12 @@ var _ = Describe("Rekor ErrorHandler", func() { Eventually(func(g Gomega) v1alpha1.RekorStatus { g.Expect(k8sClient.Get(ctx, typeNamespaceName, found)).Should(Succeed()) return found.Status - }).WithPolling(1 * time.Second).WithTimeout(5 * time.Minute).Should(And( + }).Should(And( WithTransform( func(status v1alpha1.RekorStatus) string { return meta.FindStatusCondition(status.Conditions, constants.Ready).Reason }, Not(Equal(constants.Error))), - WithTransform(func(status v1alpha1.RekorStatus) int { + WithTransform(func(status v1alpha1.RekorStatus) int64 { return status.RecoveryAttempts }, BeNumerically(">=", 1)))) Eventually(func(g Gomega) string { @@ -166,25 +166,6 @@ var _ = Describe("Rekor ErrorHandler", func() { return meta.FindStatusCondition(found.Status.Conditions, constants.Ready).Reason }).Should(Equal(constants.Initialize)) - By("Mock http client to return public key on /api/v1/log/publicKey call") - pubKeyData, err := kubernetes.GetSecretData(k8sClient, Namespace, &v1alpha1.SecretKeySelector{ - LocalObjectReference: v1alpha1.LocalObjectReference{ - Name: found.Status.Signer.KeyRef.Name, - }, - Key: "public", - }) - Expect(err).To(Succeed()) - - httpmock.SetMockTransport(http.DefaultClient, map[string]httpmock.RoundTripFunc{ - "http://rekor-server.errorhandler.svc/api/v1/log/publicKey": func(req *http.Request) *http.Response { - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewReader(pubKeyData)), - Header: make(http.Header), - } - }, - }) - deployments := &appsv1.DeploymentList{} Expect(k8sClient.List(ctx, deployments, runtimeClient.InNamespace(Namespace))).To(Succeed()) By("Move to Ready phase") @@ -195,17 +176,39 @@ var _ = Describe("Rekor ErrorHandler", func() { } // Workaround to succeed condition for Ready phase + By("Deployment should fail - public key can't be downloaded") + Eventually(func() string { + Expect(k8sClient.Get(ctx, typeNamespaceName, found)).Should(Succeed()) + condition := meta.FindStatusCondition(found.Status.Conditions, constants.Ready) + if condition == nil { + return "" + } + return condition.Reason + }).Should(Equal(constants.Error)) + + By("Fix the error. Deployment should reach Ready phase") + httpmock.SetMockTransport(server.HttpClient, map[string]httpmock.RoundTripFunc{ + "http://rekor-server.errorhandler.svc/api/v1/log/publicKey": func(req *http.Request) *http.Response { + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewReader([]byte("-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEy5wMSNagtqLsSF+zf8gBVHm2VThGP69D\ngWyhhIm/BkemPBoD/BNq+/yvD2IjsV4unLp5Lcpv4UAGAPJHL/wm+tHD1nS4QKo/\nsXJ8Ezy1K+bM5DUEilcu4hGgQ7+RCG/H\n-----END PUBLIC KEY-----"))), + Header: make(http.Header), + } + }, + }) + defer httpmock.RestoreDefaultTransport(server.HttpClient) + Eventually(func(g Gomega) v1alpha1.RekorStatus { g.Expect(k8sClient.Get(ctx, typeNamespaceName, found)).Should(Succeed()) return found.Status - }).WithPolling(1 * time.Second).WithTimeout(5 * time.Minute).Should(And( + }).Should(And( WithTransform( func(status v1alpha1.RekorStatus) bool { return meta.IsStatusConditionTrue(status.Conditions, constants.Ready) }, Equal(true)), - WithTransform(func(status v1alpha1.RekorStatus) int { + WithTransform(func(status v1alpha1.RekorStatus) int64 { return status.RecoveryAttempts - }, Equal(0)))) + }, BeNumerically("==", 0)))) By("Pregenerated resources are reused") Expect(signerName).To(Equal(found.Status.Signer.KeyRef.Name)) diff --git a/internal/controller/rekor/rekor_hot_update_test.go b/internal/controller/rekor/rekor_hot_update_test.go index 568ba0fe2..6f08c4b81 100644 --- a/internal/controller/rekor/rekor_hot_update_test.go +++ b/internal/controller/rekor/rekor_hot_update_test.go @@ -19,12 +19,13 @@ package rekor import ( "bytes" "context" - "github.com/securesign/operator/internal/controller/rekor/actions/server" - httpmock "github.com/securesign/operator/internal/testing/http" "io" "net/http" "time" + "github.com/securesign/operator/internal/controller/rekor/actions/server" + httpmock "github.com/securesign/operator/internal/testing/http" + "github.com/securesign/operator/internal/controller/common/utils" "github.com/securesign/operator/api/v1alpha1" @@ -134,25 +135,6 @@ var _ = Describe("Rekor hot update test", func() { }).Should(Not(BeNil())) Expect(k8sClient.Get(ctx, types.NamespacedName{Name: found.Status.Signer.KeyRef.Name, Namespace: Namespace}, &corev1.Secret{})).Should(Succeed()) - By("Mock http client to return public key on /api/v1/log/publicKey call") - pubKeyData, err := kubernetes.GetSecretData(k8sClient, Namespace, &v1alpha1.SecretKeySelector{ - LocalObjectReference: v1alpha1.LocalObjectReference{ - Name: found.Status.Signer.KeyRef.Name, - }, - Key: "public", - }) - Expect(err).To(Succeed()) - - httpmock.SetMockTransport(http.DefaultClient, map[string]httpmock.RoundTripFunc{ - "http://rekor-server." + Namespace + ".svc/api/v1/log/publicKey": func(req *http.Request) *http.Response { - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewReader(pubKeyData)), - Header: make(http.Header), - } - }, - }) - By("Waiting until Rekor instance is Initialization") Eventually(func(g Gomega) string { found := &v1alpha1.Rekor{} @@ -161,6 +143,16 @@ var _ = Describe("Rekor hot update test", func() { return meta.FindStatusCondition(found.Status.Conditions, constants.Ready).Reason }).Should(Equal(constants.Initialize)) + httpmock.SetMockTransport(server.HttpClient, map[string]httpmock.RoundTripFunc{ + "http://rekor-server.update.svc/api/v1/log/publicKey": func(req *http.Request) *http.Response { + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewReader([]byte("-----BEGIN PUBLIC KEY-----\nMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEy5wMSNagtqLsSF+zf8gBVHm2VThGP69D\ngWyhhIm/BkemPBoD/BNq+/yvD2IjsV4unLp5Lcpv4UAGAPJHL/wm+tHD1nS4QKo/\nsXJ8Ezy1K+bM5DUEilcu4hGgQ7+RCG/H\n-----END PUBLIC KEY-----"))), + Header: make(http.Header), + } + }, + }) + defer httpmock.RestoreDefaultTransport(server.HttpClient) deployments := &appsv1.DeploymentList{} Expect(k8sClient.List(ctx, deployments, runtimeClient.InNamespace(Namespace))).To(Succeed()) By("Move to Ready phase") @@ -171,13 +163,6 @@ var _ = Describe("Rekor hot update test", func() { } // Workaround to succeed condition for Ready phase - By("Rekor public key secret created") - Eventually(func(g Gomega) { - scr := &corev1.SecretList{} - g.Expect(k8sClient.List(ctx, scr, runtimeClient.InNamespace(Namespace), runtimeClient.MatchingLabels{server.RekorPubLabel: "public"})).Should(Succeed()) - g.Expect(scr.Items).Should(HaveLen(1)) - }).Should(Succeed()) - By("Waiting until Rekor instance is Ready") Eventually(func(g Gomega) bool { found := &v1alpha1.Rekor{} @@ -185,6 +170,13 @@ var _ = Describe("Rekor hot update test", func() { return meta.IsStatusConditionTrue(found.Status.Conditions, constants.Ready) }).Should(BeTrue()) + By("Rekor public key secret created") + Eventually(func(g Gomega) { + scr := &corev1.SecretList{} + g.Expect(k8sClient.List(ctx, scr, runtimeClient.InNamespace(Namespace), runtimeClient.MatchingLabels{server.RekorPubLabel: "public"})).Should(Succeed()) + g.Expect(scr.Items).Should(HaveLen(1)) + }).Should(Succeed()) + By("Save the Deployment configuration") deployment := &appsv1.Deployment{} Expect(k8sClient.Get(ctx, types.NamespacedName{Name: actions.ServerDeploymentName, Namespace: Namespace}, deployment)).Should(Succeed()) @@ -202,19 +194,9 @@ var _ = Describe("Rekor hot update test", func() { return k8sClient.Update(ctx, found) }).Should(Succeed()) - By("Move to CreatingPhase by creating trillian service") + By("Move to CreatingPhase by creating new private key") Expect(k8sClient.Create(ctx, kubernetes.CreateSecret("key-secret", Namespace, map[string][]byte{"private": []byte("fake")}, constants.LabelsFor(actions.ServerComponentName, actions.ServerDeploymentName, instance.Name)))).To(Succeed()) - httpmock.SetMockTransport(http.DefaultClient, map[string]httpmock.RoundTripFunc{ - "http://rekor-server." + Namespace + ".svc/api/v1/log/publicKey": func(req *http.Request) *http.Response { - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewReader([]byte("newPublicKey"))), - Header: make(http.Header), - } - }, - }) - By("Secret key is resolved") Eventually(func(g Gomega) *v1alpha1.SecretKeySelector { g.Expect(k8sClient.Get(ctx, typeNamespaceName, found)).Should(Succeed()) @@ -225,6 +207,17 @@ var _ = Describe("Rekor hot update test", func() { return found.Status.Signer.KeyRef.Name }).Should(Equal("key-secret")) + httpmock.SetMockTransport(server.HttpClient, map[string]httpmock.RoundTripFunc{ + "http://rekor-server.update.svc/api/v1/log/publicKey": func(req *http.Request) *http.Response { + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewReader([]byte("newPublicKey"))), + Header: make(http.Header), + } + }, + }) + defer httpmock.RestoreDefaultTransport(server.HttpClient) + By("Rekor deployment is updated") Eventually(func() bool { updated := &appsv1.Deployment{} diff --git a/internal/controller/securesign/actions/ensure_ctlog.go b/internal/controller/securesign/actions/ensure_ctlog.go index 221d97fd6..45f3f3505 100644 --- a/internal/controller/securesign/actions/ensure_ctlog.go +++ b/internal/controller/securesign/actions/ensure_ctlog.go @@ -2,6 +2,7 @@ package actions import ( "context" + "github.com/securesign/operator/internal/controller/annotations" rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1" @@ -45,7 +46,7 @@ func (i ctlogAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Secures ctlog.Spec = instance.Spec.Ctlog if err = controllerutil.SetControllerReference(instance, ctlog, i.Client.Scheme()); err != nil { - return i.Failed(err) + return i.Error(err) } if updated, err = i.Ensure(ctx, ctlog); err != nil { @@ -55,7 +56,7 @@ func (i ctlogAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Secures Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, err, instance) + return i.ErrorWithStatusUpdate(ctx, err, instance) } if updated { @@ -74,7 +75,7 @@ func (i ctlogAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Secures func (i ctlogAction) CopyStatus(ctx context.Context, ok client.ObjectKey, instance *rhtasv1alpha1.Securesign) *action.Result { ctl := &rhtasv1alpha1.CTlog{} if err := i.Client.Get(ctx, ok, ctl); err != nil { - return i.Failed(err) + return i.Error(err) } objectStatus := meta.FindStatusCondition(ctl.Status.Conditions, constants.Ready) if objectStatus == nil { @@ -91,3 +92,11 @@ func (i ctlogAction) CopyStatus(ctx context.Context, ok client.ObjectKey, instan } return i.Continue() } + +func (i ctlogAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Securesign) bool { + return false +} + +func (i ctlogAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Securesign) *action.Result { + return i.Continue() +} diff --git a/internal/controller/securesign/actions/ensure_fulcio.go b/internal/controller/securesign/actions/ensure_fulcio.go index 0495a5d4d..cafb3216d 100644 --- a/internal/controller/securesign/actions/ensure_fulcio.go +++ b/internal/controller/securesign/actions/ensure_fulcio.go @@ -2,6 +2,7 @@ package actions import ( "context" + "github.com/securesign/operator/internal/controller/annotations" rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1" @@ -45,7 +46,7 @@ func (i fulcioAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Secure fulcio.Spec = instance.Spec.Fulcio if err = controllerutil.SetControllerReference(instance, fulcio, i.Client.Scheme()); err != nil { - return i.Failed(err) + return i.Error(err) } if updated, err = i.Ensure(ctx, fulcio); err != nil { @@ -55,7 +56,7 @@ func (i fulcioAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Secure Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, err, instance) + return i.ErrorWithStatusUpdate(ctx, err, instance) } if updated { @@ -74,7 +75,7 @@ func (i fulcioAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Secure func (i fulcioAction) CopyStatus(ctx context.Context, ok client.ObjectKey, instance *rhtasv1alpha1.Securesign) *action.Result { object := &rhtasv1alpha1.Fulcio{} if err := i.Client.Get(ctx, ok, object); err != nil { - return i.Failed(err) + return i.Error(err) } objectStatus := meta.FindStatusCondition(object.Status.Conditions, constants.Ready) if objectStatus == nil { @@ -94,3 +95,11 @@ func (i fulcioAction) CopyStatus(ctx context.Context, ok client.ObjectKey, insta } return i.Continue() } + +func (i fulcioAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Securesign) bool { + return false +} + +func (i fulcioAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Securesign) *action.Result { + return i.Continue() +} diff --git a/internal/controller/securesign/actions/ensure_rekor.go b/internal/controller/securesign/actions/ensure_rekor.go index 9d39530c3..b66315bb7 100644 --- a/internal/controller/securesign/actions/ensure_rekor.go +++ b/internal/controller/securesign/actions/ensure_rekor.go @@ -2,6 +2,7 @@ package actions import ( "context" + "github.com/securesign/operator/internal/controller/annotations" rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1" @@ -44,7 +45,7 @@ func (i rekorAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Secures rekor.Spec = instance.Spec.Rekor if err = controllerutil.SetControllerReference(instance, rekor, i.Client.Scheme()); err != nil { - return i.Failed(err) + return i.Error(err) } if updated, err = i.Ensure(ctx, rekor); err != nil { @@ -54,7 +55,7 @@ func (i rekorAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Secures Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, err, instance) + return i.ErrorWithStatusUpdate(ctx, err, instance) } if updated { @@ -73,7 +74,7 @@ func (i rekorAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Secures func (i rekorAction) CopyStatus(ctx context.Context, ok client.ObjectKey, instance *rhtasv1alpha1.Securesign) *action.Result { object := &rhtasv1alpha1.Rekor{} if err := i.Client.Get(ctx, ok, object); err != nil { - return i.Failed(err) + return i.Error(err) } objectStatus := meta.FindStatusCondition(object.Status.Conditions, constants.Ready) if objectStatus == nil { @@ -93,3 +94,11 @@ func (i rekorAction) CopyStatus(ctx context.Context, ok client.ObjectKey, instan } return i.Continue() } + +func (i rekorAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Securesign) bool { + return false +} + +func (i rekorAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Securesign) *action.Result { + return i.Continue() +} diff --git a/internal/controller/securesign/actions/ensure_trillian.go b/internal/controller/securesign/actions/ensure_trillian.go index 30f8c4827..deff9a59f 100644 --- a/internal/controller/securesign/actions/ensure_trillian.go +++ b/internal/controller/securesign/actions/ensure_trillian.go @@ -2,6 +2,7 @@ package actions import ( "context" + "github.com/securesign/operator/internal/controller/annotations" rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1" @@ -44,7 +45,7 @@ func (i trillianAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Secu trillian.Spec = instance.Spec.Trillian if err = controllerutil.SetControllerReference(instance, trillian, i.Client.Scheme()); err != nil { - return i.Failed(err) + return i.Error(err) } if updated, err = i.Ensure(ctx, trillian); err != nil { @@ -54,7 +55,7 @@ func (i trillianAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Secu Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, err, instance) + return i.ErrorWithStatusUpdate(ctx, err, instance) } if updated { @@ -73,7 +74,7 @@ func (i trillianAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Secu func (i trillianAction) CopyStatus(ctx context.Context, ok client.ObjectKey, instance *rhtasv1alpha1.Securesign) *action.Result { object := &rhtasv1alpha1.Trillian{} if err := i.Client.Get(ctx, ok, object); err != nil { - return i.Failed(err) + return i.Error(err) } objectStatus := meta.FindStatusCondition(object.Status.Conditions, constants.Ready) if objectStatus == nil { @@ -90,3 +91,11 @@ func (i trillianAction) CopyStatus(ctx context.Context, ok client.ObjectKey, ins } return i.Continue() } + +func (i trillianAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Securesign) bool { + return false +} + +func (i trillianAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Securesign) *action.Result { + return i.Continue() +} diff --git a/internal/controller/securesign/actions/ensure_tuf.go b/internal/controller/securesign/actions/ensure_tuf.go index 884a7ef30..87e0827ca 100644 --- a/internal/controller/securesign/actions/ensure_tuf.go +++ b/internal/controller/securesign/actions/ensure_tuf.go @@ -2,6 +2,7 @@ package actions import ( "context" + "github.com/securesign/operator/internal/controller/annotations" rhtasv1alpha1 "github.com/securesign/operator/api/v1alpha1" @@ -45,7 +46,7 @@ func (i tufAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Securesig tuf.Spec = instance.Spec.Tuf if err = controllerutil.SetControllerReference(instance, tuf, i.Client.Scheme()); err != nil { - return i.Failed(err) + return i.Error(err) } if updated, err = i.Ensure(ctx, tuf); err != nil { @@ -55,7 +56,7 @@ func (i tufAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Securesig Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, err, instance) + return i.ErrorWithStatusUpdate(ctx, err, instance) } if updated { @@ -74,7 +75,7 @@ func (i tufAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Securesig func (i tufAction) CopyStatus(ctx context.Context, ok client.ObjectKey, instance *rhtasv1alpha1.Securesign) *action.Result { object := &rhtasv1alpha1.Tuf{} if err := i.Client.Get(ctx, ok, object); err != nil { - return i.Failed(err) + return i.Error(err) } objectStatus := meta.FindStatusCondition(object.Status.Conditions, constants.Ready) if objectStatus == nil { @@ -94,3 +95,11 @@ func (i tufAction) CopyStatus(ctx context.Context, ok client.ObjectKey, instance } return i.Continue() } + +func (i tufAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Securesign) bool { + return false +} + +func (i tufAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Securesign) *action.Result { + return i.Continue() +} diff --git a/internal/controller/securesign/actions/initialize_status.go b/internal/controller/securesign/actions/initialize_status.go index c4c66f572..f9aade94a 100644 --- a/internal/controller/securesign/actions/initialize_status.go +++ b/internal/controller/securesign/actions/initialize_status.go @@ -36,3 +36,11 @@ func (i initializeStatus) Handle(ctx context.Context, instance *rhtasv1alpha1.Se } return i.StatusUpdate(ctx, instance) } + +func (i initializeStatus) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Securesign) bool { + return false +} + +func (i initializeStatus) HandleError(_ context.Context, _ *rhtasv1alpha1.Securesign) *action.Result { + return i.Continue() +} diff --git a/internal/controller/securesign/actions/rbac.go b/internal/controller/securesign/actions/rbac.go index 55b562e4f..ff35b422a 100644 --- a/internal/controller/securesign/actions/rbac.go +++ b/internal/controller/securesign/actions/rbac.go @@ -61,7 +61,7 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Securesi }, } if err = controllerutil.SetControllerReference(instance, serviceAccount, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for serviceAccount: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for serviceAccount: %w", err)) } if _, err = i.Ensure(ctx, serviceAccount); err != nil { meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ @@ -76,7 +76,7 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Securesi Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create serviceAccount: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create serviceAccount: %w", err), instance) } openshiftMonitoringSBJRole := kubernetes.CreateRole( @@ -109,7 +109,7 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Securesi Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create openshift-monitoring role for SBJ: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create openshift-monitoring role for SBJ: %w", err), instance) } openshiftMonitoringSBJRoleBinding := kubernetes.CreateRoleBinding( @@ -137,7 +137,7 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Securesi Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create openshift-monitoring role binding for SBJ: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create openshift-monitoring role binding for SBJ: %w", err), instance) } openshiftMonitoringClusterRoleBinding := kubernetes.CreateClusterRoleBinding( @@ -164,7 +164,7 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Securesi Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create monitoring ClusterRoleBinding for SBJ: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create monitoring ClusterRoleBinding for SBJ: %w", err), instance) } openshiftConsoleSBJRole := kubernetes.CreateClusterRole( @@ -197,7 +197,7 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Securesi Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create openshift-console ClusterRole for SBJ: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create openshift-console ClusterRole for SBJ: %w", err), instance) } openshiftConsoleSBJRolebinding := kubernetes.CreateClusterRoleBinding( @@ -224,8 +224,16 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Securesi Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create openshift-console ClusterRoleBinding for SBJ: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create openshift-console ClusterRoleBinding for SBJ: %w", err), instance) } return i.Continue() } + +func (i rbacAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Securesign) bool { + return false +} + +func (i rbacAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Securesign) *action.Result { + return i.Continue() +} diff --git a/internal/controller/securesign/actions/segment_backup_cronjob.go b/internal/controller/securesign/actions/segment_backup_cronjob.go index 7f5ab9393..d7d46237b 100644 --- a/internal/controller/securesign/actions/segment_backup_cronjob.go +++ b/internal/controller/securesign/actions/segment_backup_cronjob.go @@ -52,7 +52,7 @@ func (i segmentBackupCronJob) Handle(ctx context.Context, instance *rhtasv1alpha ) if _, err := cron.ParseStandard(AnalyiticsCronSchedule); err != nil { - return i.Failed(fmt.Errorf("could not create segment backuup cron job due to errors with parsing the cron schedule: %w", err)) + return i.Error(fmt.Errorf("could not create segment backuup cron job due to errors with parsing the cron schedule: %w", err)) } labels := constants.LabelsFor(SegmentBackupCronJobName, SegmentBackupCronJobName, instance.Name) @@ -101,7 +101,7 @@ func (i segmentBackupCronJob) Handle(ctx context.Context, instance *rhtasv1alpha } if err = controllerutil.SetControllerReference(instance, segmentBackupCronJob, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for segment backup cron job: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for segment backup cron job: %w", err)) } if updated, err = i.Ensure(ctx, segmentBackupCronJob); err != nil { @@ -117,7 +117,7 @@ func (i segmentBackupCronJob) Handle(ctx context.Context, instance *rhtasv1alpha Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create segment backup cron job: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create segment backup cron job: %w", err), instance) } if updated { @@ -132,3 +132,11 @@ func (i segmentBackupCronJob) Handle(ctx context.Context, instance *rhtasv1alpha return i.Continue() } } + +func (i segmentBackupCronJob) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Securesign) bool { + return false +} + +func (i segmentBackupCronJob) HandleError(_ context.Context, _ *rhtasv1alpha1.Securesign) *action.Result { + return i.Continue() +} diff --git a/internal/controller/securesign/actions/segment_backup_job.go b/internal/controller/securesign/actions/segment_backup_job.go index f46cc4c3a..0cb536b6b 100644 --- a/internal/controller/securesign/actions/segment_backup_job.go +++ b/internal/controller/securesign/actions/segment_backup_job.go @@ -95,12 +95,12 @@ func (i segmentBackupJob) Handle(ctx context.Context, instance *rhtasv1alpha1.Se job := kubernetes.CreateJob(instance.Namespace, SegmentBackupJobName, labels, constants.SegmentBackupImage, SegmentRBACName, parallelism, completions, activeDeadlineSeconds, backoffLimit, command, env) if err = ctrl.SetControllerReference(instance, job, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for Job: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for Job: %w", err)) } _, err = i.Ensure(ctx, job) if err != nil { - return i.Failed(fmt.Errorf("failed to Ensure the job: %w", err)) + return i.Error(fmt.Errorf("failed to Ensure the job: %w", err)) } meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ @@ -112,3 +112,11 @@ func (i segmentBackupJob) Handle(ctx context.Context, instance *rhtasv1alpha1.Se return i.Continue() } + +func (i segmentBackupJob) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Securesign) bool { + return false +} + +func (i segmentBackupJob) HandleError(_ context.Context, _ *rhtasv1alpha1.Securesign) *action.Result { + return i.Continue() +} diff --git a/internal/controller/securesign/actions/update_status.go b/internal/controller/securesign/actions/update_status.go index 7c9f15635..2eb7952ac 100644 --- a/internal/controller/securesign/actions/update_status.go +++ b/internal/controller/securesign/actions/update_status.go @@ -67,3 +67,11 @@ func sortByStatus(conditions []v1.Condition) []string { }) return sorted } + +func (i updateStatusAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Securesign) bool { + return false +} + +func (i updateStatusAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Securesign) *action.Result { + return i.Continue() +} diff --git a/internal/controller/trillian/actions/db/deployment.go b/internal/controller/trillian/actions/db/deployment.go index dda9d075d..d8dc6fcca 100644 --- a/internal/controller/trillian/actions/db/deployment.go +++ b/internal/controller/trillian/actions/db/deployment.go @@ -63,10 +63,10 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Trilli Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create Trillian DB: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create Trillian DB: %w", err), instance) } if err = controllerutil.SetControllerReference(instance, db, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for DB Deployment: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for DB Deployment: %w", err)) } if updated, err = i.Ensure(ctx, db); err != nil { @@ -82,7 +82,7 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Trilli Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create Trillian DB: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create Trillian DB: %w", err), instance) } if updated { @@ -96,5 +96,12 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Trilli } else { return i.Continue() } +} + +func (i deployAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) bool { + return false +} +func (i deployAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) *action.Result { + return i.Continue() } diff --git a/internal/controller/trillian/actions/db/handle_secret.go b/internal/controller/trillian/actions/db/handle_secret.go index 025a35e03..d6b4d0705 100644 --- a/internal/controller/trillian/actions/db/handle_secret.go +++ b/internal/controller/trillian/actions/db/handle_secret.go @@ -60,7 +60,7 @@ func (i handleSecretAction) Handle(ctx context.Context, instance *rhtasv1alpha1. dbSecret := i.createDbSecret(instance.Namespace, dbLabels) if err = controllerutil.SetControllerReference(instance, dbSecret, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for secret: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for secret: %w", err)) } // no watch on secret - continue if no error @@ -77,7 +77,7 @@ func (i handleSecretAction) Handle(ctx context.Context, instance *rhtasv1alpha1. Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create DB secret: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create DB secret: %w", err), instance) } instance.Status.Db.DatabaseSecretRef = &rhtasv1alpha1.LocalObjectReference{ @@ -108,3 +108,11 @@ func (i handleSecretAction) createDbSecret(namespace string, labels map[string]s }, } } + +func (i handleSecretAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) bool { + return false +} + +func (i handleSecretAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) *action.Result { + return i.Continue() +} diff --git a/internal/controller/trillian/actions/db/initialize.go b/internal/controller/trillian/actions/db/initialize.go index ee4eb1c63..e8202fb48 100644 --- a/internal/controller/trillian/actions/db/initialize.go +++ b/internal/controller/trillian/actions/db/initialize.go @@ -35,7 +35,7 @@ func (i initializeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tr labels := constants.LabelsForComponent(actions.DbComponentName, instance.Name) ok, err := commonUtils.DeploymentIsRunning(ctx, i.Client, instance.Namespace, labels) if err != nil { - return i.Failed(err) + return i.Error(err) } if !ok { i.Logger.Info("Waiting for deployment") @@ -52,3 +52,11 @@ func (i initializeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tr Status: metav1.ConditionTrue, Reason: constants.Ready}) return i.StatusUpdate(ctx, instance) } + +func (i initializeAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) bool { + return false +} + +func (i initializeAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) *action.Result { + return i.Continue() +} diff --git a/internal/controller/trillian/actions/db/pvc.go b/internal/controller/trillian/actions/db/pvc.go index 19a19bcf2..50b53ef90 100644 --- a/internal/controller/trillian/actions/db/pvc.go +++ b/internal/controller/trillian/actions/db/pvc.go @@ -44,7 +44,7 @@ func (i createPvcAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tri } if instance.Spec.Db.Pvc.Size == nil { - return i.Failed(fmt.Errorf("PVC size is not set")) + return i.Error(fmt.Errorf("PVC size is not set")) } // PVC does not exist, create a new one @@ -52,7 +52,7 @@ func (i createPvcAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tri pvc := k8sutils.CreatePVC(instance.Namespace, actions.DbPvcName, *instance.Spec.Db.Pvc.Size, instance.Spec.Db.Pvc.StorageClass, constants.LabelsFor(actions.DbComponentName, actions.DbDeploymentName, instance.Name)) if !utils.OptionalBool(instance.Spec.Db.Pvc.Retain) { if err = controllerutil.SetControllerReference(instance, pvc, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for PVC: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for PVC: %w", err)) } } if _, err = i.Ensure(ctx, pvc); err != nil { @@ -68,10 +68,18 @@ func (i createPvcAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tri Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create DB PVC: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create DB PVC: %w", err), instance) } i.Recorder.Event(instance, v1.EventTypeNormal, "PersistentVolumeCreated", "New PersistentVolume created") instance.Status.Db.Pvc.Name = pvc.Name return i.StatusUpdate(ctx, instance) } + +func (i createPvcAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) bool { + return false +} + +func (i createPvcAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) *action.Result { + return i.Continue() +} diff --git a/internal/controller/trillian/actions/db/svc.go b/internal/controller/trillian/actions/db/svc.go index 575b9326d..f94a740f5 100644 --- a/internal/controller/trillian/actions/db/svc.go +++ b/internal/controller/trillian/actions/db/svc.go @@ -45,7 +45,7 @@ func (i createServiceAction) Handle(ctx context.Context, instance *rhtasv1alpha1 mysql := k8sutils.CreateService(instance.Namespace, host, host, port, port, labels) if err = controllerutil.SetControllerReference(instance, mysql, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for DB service: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for DB service: %w", err)) } if updated, err = i.Ensure(ctx, mysql); err != nil { @@ -61,7 +61,7 @@ func (i createServiceAction) Handle(ctx context.Context, instance *rhtasv1alpha1 Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create Trillian DB service: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create Trillian DB service: %w", err), instance) } if updated { @@ -76,3 +76,11 @@ func (i createServiceAction) Handle(ctx context.Context, instance *rhtasv1alpha1 return i.Continue() } } + +func (i createServiceAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) bool { + return false +} + +func (i createServiceAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) *action.Result { + return i.Continue() +} diff --git a/internal/controller/trillian/actions/initialize.go b/internal/controller/trillian/actions/initialize.go index 39e29b898..dee182879 100644 --- a/internal/controller/trillian/actions/initialize.go +++ b/internal/controller/trillian/actions/initialize.go @@ -38,3 +38,11 @@ func (i initializeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tr return i.Requeue() } + +func (i initializeAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) bool { + return false +} + +func (i initializeAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) *action.Result { + return i.Continue() +} diff --git a/internal/controller/trillian/actions/logserver/deployment.go b/internal/controller/trillian/actions/logserver/deployment.go index bea9cf170..01b03a9ce 100644 --- a/internal/controller/trillian/actions/logserver/deployment.go +++ b/internal/controller/trillian/actions/logserver/deployment.go @@ -58,11 +58,11 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Trilli Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create Trillian server: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create Trillian server: %w", err), instance) } if err = controllerutil.SetControllerReference(instance, server, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for server: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for server: %w", err)) } if updated, err = i.Ensure(ctx, server); err != nil { @@ -78,7 +78,7 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Trilli Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create Trillian server: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create Trillian server: %w", err), instance) } if updated { @@ -93,3 +93,11 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Trilli return i.Continue() } } + +func (i deployAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) bool { + return false +} + +func (i deployAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) *action.Result { + return i.Continue() +} diff --git a/internal/controller/trillian/actions/logserver/initialize.go b/internal/controller/trillian/actions/logserver/initialize.go index e03d6ca86..689d34ad0 100644 --- a/internal/controller/trillian/actions/logserver/initialize.go +++ b/internal/controller/trillian/actions/logserver/initialize.go @@ -33,7 +33,7 @@ func (i initializeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tr labels := constants.LabelsForComponent(actions.LogServerComponentName, instance.Name) ok, err := commonUtils.DeploymentIsRunning(ctx, i.Client, instance.Namespace, labels) if err != nil { - return i.Failed(err) + return i.Error(err) } if !ok { i.Logger.Info("Waiting for deployment") @@ -50,3 +50,11 @@ func (i initializeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tr Status: metav1.ConditionTrue, Reason: constants.Ready}) return i.StatusUpdate(ctx, instance) } + +func (i initializeAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) bool { + return false +} + +func (i initializeAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) *action.Result { + return i.Continue() +} diff --git a/internal/controller/trillian/actions/logserver/monitoring.go b/internal/controller/trillian/actions/logserver/monitoring.go index fa8db4305..2fae3b011 100644 --- a/internal/controller/trillian/actions/logserver/monitoring.go +++ b/internal/controller/trillian/actions/logserver/monitoring.go @@ -53,7 +53,7 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tr ) if err = controllerutil.SetControllerReference(instance, role, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for role: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for role: %w", err)) } if _, err = i.Ensure(ctx, role); err != nil { @@ -69,7 +69,7 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tr Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create monitoring role: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create monitoring role: %w", err), instance) } roleBinding := kubernetes.CreateRoleBinding( @@ -86,7 +86,7 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tr }, ) if err = controllerutil.SetControllerReference(instance, roleBinding, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for role: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for role: %w", err)) } if _, err = i.Ensure(ctx, roleBinding); err != nil { @@ -102,7 +102,7 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tr Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create monitoring RoleBinding: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create monitoring RoleBinding: %w", err), instance) } serviceMonitor := kubernetes.CreateServiceMonitor( @@ -120,7 +120,7 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tr ) if err = controllerutil.SetControllerReference(instance, serviceMonitor, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for serviceMonitor: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for serviceMonitor: %w", err)) } if _, err = i.Ensure(ctx, serviceMonitor); err != nil { @@ -136,9 +136,17 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tr Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create serviceMonitor: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create serviceMonitor: %w", err), instance) } // monitors & RBAC are not watched - do not need to re-enqueue return i.Continue() } + +func (i monitoringAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) bool { + return false +} + +func (i monitoringAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) *action.Result { + return i.Continue() +} diff --git a/internal/controller/trillian/actions/logserver/service.go b/internal/controller/trillian/actions/logserver/service.go index 774911f48..3627ddf59 100644 --- a/internal/controller/trillian/actions/logserver/service.go +++ b/internal/controller/trillian/actions/logserver/service.go @@ -54,7 +54,7 @@ func (i createServiceAction) Handle(ctx context.Context, instance *rhtasv1alpha1 } if err = controllerutil.SetControllerReference(instance, logserverService, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for logserver Service: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for logserver Service: %w", err)) } if updated, err = i.Ensure(ctx, logserverService); err != nil { @@ -70,7 +70,7 @@ func (i createServiceAction) Handle(ctx context.Context, instance *rhtasv1alpha1 Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create logserver Service: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create logserver Service: %w", err), instance) } if updated { @@ -84,5 +84,12 @@ func (i createServiceAction) Handle(ctx context.Context, instance *rhtasv1alpha1 } else { return i.Continue() } +} + +func (i createServiceAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) bool { + return false +} +func (i createServiceAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) *action.Result { + return i.Continue() } diff --git a/internal/controller/trillian/actions/logsigner/deployment.go b/internal/controller/trillian/actions/logsigner/deployment.go index 4d0e60f8e..12e17afd3 100644 --- a/internal/controller/trillian/actions/logsigner/deployment.go +++ b/internal/controller/trillian/actions/logsigner/deployment.go @@ -3,6 +3,7 @@ package logsigner import ( "context" "fmt" + "github.com/securesign/operator/internal/controller/common/utils" "github.com/securesign/operator/internal/controller/common/action" @@ -56,11 +57,11 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Trilli Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create Trillian LogSigner: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create Trillian LogSigner: %w", err), instance) } if err = controllerutil.SetControllerReference(instance, signer, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for LogSigner deployment: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for LogSigner deployment: %w", err)) } if updated, err = i.Ensure(ctx, signer); err != nil { @@ -76,7 +77,7 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Trilli Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create Trillian LogSigner deployment: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create Trillian LogSigner deployment: %w", err), instance) } if updated { @@ -91,3 +92,11 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Trilli return i.Continue() } } + +func (i deployAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) bool { + return false +} + +func (i deployAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) *action.Result { + return i.Continue() +} diff --git a/internal/controller/trillian/actions/logsigner/initialize.go b/internal/controller/trillian/actions/logsigner/initialize.go index 70ddace6b..e3d61b359 100644 --- a/internal/controller/trillian/actions/logsigner/initialize.go +++ b/internal/controller/trillian/actions/logsigner/initialize.go @@ -33,7 +33,7 @@ func (i initializeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tr labels := constants.LabelsForComponent(actions.LogSignerComponentName, instance.Name) ok, err := commonUtils.DeploymentIsRunning(ctx, i.Client, instance.Namespace, labels) if err != nil { - return i.Failed(err) + return i.Error(err) } if !ok { i.Logger.Info("Waiting for deployment") @@ -49,3 +49,11 @@ func (i initializeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tr Status: metav1.ConditionTrue, Reason: constants.Ready}) return i.StatusUpdate(ctx, instance) } + +func (i initializeAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) bool { + return false +} + +func (i initializeAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) *action.Result { + return i.Continue() +} diff --git a/internal/controller/trillian/actions/logsigner/monitoring.go b/internal/controller/trillian/actions/logsigner/monitoring.go index 503e65a6a..a1a47b181 100644 --- a/internal/controller/trillian/actions/logsigner/monitoring.go +++ b/internal/controller/trillian/actions/logsigner/monitoring.go @@ -53,7 +53,7 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tr ) if err = controllerutil.SetControllerReference(instance, role, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for role: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for role: %w", err)) } if _, err = i.Ensure(ctx, role); err != nil { @@ -69,7 +69,7 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tr Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create monitoring role: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create monitoring role: %w", err), instance) } roleBinding := kubernetes.CreateRoleBinding( @@ -86,7 +86,7 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tr }, ) if err = controllerutil.SetControllerReference(instance, roleBinding, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for role: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for role: %w", err)) } if _, err = i.Ensure(ctx, roleBinding); err != nil { @@ -102,7 +102,7 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tr Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create monitoring RoleBinding: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create monitoring RoleBinding: %w", err), instance) } serviceMonitor := kubernetes.CreateServiceMonitor( @@ -120,7 +120,7 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tr ) if err = controllerutil.SetControllerReference(instance, serviceMonitor, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for serviceMonitor: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for serviceMonitor: %w", err)) } if _, err = i.Ensure(ctx, serviceMonitor); err != nil { @@ -136,9 +136,17 @@ func (i monitoringAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tr Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create serviceMonitor: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create serviceMonitor: %w", err), instance) } // monitors & RBAC are not watched - do not need to re-enqueue return i.Continue() } + +func (i monitoringAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) bool { + return false +} + +func (i monitoringAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) *action.Result { + return i.Continue() +} diff --git a/internal/controller/trillian/actions/logsigner/service.go b/internal/controller/trillian/actions/logsigner/service.go index a47b8bc2b..31f6f1331 100644 --- a/internal/controller/trillian/actions/logsigner/service.go +++ b/internal/controller/trillian/actions/logsigner/service.go @@ -53,7 +53,7 @@ func (i createServiceAction) Handle(ctx context.Context, instance *rhtasv1alpha1 } if err = controllerutil.SetControllerReference(instance, logsignerService, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for logsigner Service: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for logsigner Service: %w", err)) } if updated, err = i.Ensure(ctx, logsignerService); err != nil { @@ -69,7 +69,7 @@ func (i createServiceAction) Handle(ctx context.Context, instance *rhtasv1alpha1 Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create logsigner Service: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create logsigner Service: %w", err), instance) } if updated { @@ -83,5 +83,12 @@ func (i createServiceAction) Handle(ctx context.Context, instance *rhtasv1alpha1 } else { return i.Continue() } +} + +func (i createServiceAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) bool { + return false +} +func (i createServiceAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) *action.Result { + return i.Continue() } diff --git a/internal/controller/trillian/actions/rbac.go b/internal/controller/trillian/actions/rbac.go index adf7f5633..7844ea6c3 100644 --- a/internal/controller/trillian/actions/rbac.go +++ b/internal/controller/trillian/actions/rbac.go @@ -47,7 +47,7 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Trillian } if err = ctrl.SetControllerReference(instance, sa, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controll reference for SA: %w", err)) + return i.Error(fmt.Errorf("could not set controll reference for SA: %w", err)) } // don't re-enqueue for RBAC in any case (except failure) _, err = i.Ensure(ctx, sa) @@ -58,7 +58,7 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Trillian Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create SA: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create SA: %w", err), instance) } role := kubernetes.CreateRole(instance.Namespace, RBACName, labels, []rbacv1.PolicyRule{ { @@ -74,7 +74,7 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Trillian }) if err = ctrl.SetControllerReference(instance, role, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controll reference for role: %w", err)) + return i.Error(fmt.Errorf("could not set controll reference for role: %w", err)) } _, err = i.Ensure(ctx, role) if err != nil { @@ -84,7 +84,7 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Trillian Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create Role: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create Role: %w", err), instance) } rb := kubernetes.CreateRoleBinding(instance.Namespace, RBACName, labels, rbacv1.RoleRef{ APIGroup: v1.SchemeGroupVersion.Group, @@ -96,7 +96,7 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Trillian }) if err = ctrl.SetControllerReference(instance, rb, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controll reference for roleBinding: %w", err)) + return i.Error(fmt.Errorf("could not set controll reference for roleBinding: %w", err)) } _, err = i.Ensure(ctx, rb) if err != nil { @@ -106,7 +106,15 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Trillian Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create RoleBinding: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create RoleBinding: %w", err), instance) } return i.Continue() } + +func (i rbacAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) bool { + return false +} + +func (i rbacAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Trillian) *action.Result { + return i.Continue() +} diff --git a/internal/controller/tuf/actions/deployment.go b/internal/controller/tuf/actions/deployment.go index d7cace5a1..c2927c4bd 100644 --- a/internal/controller/tuf/actions/deployment.go +++ b/internal/controller/tuf/actions/deployment.go @@ -41,7 +41,7 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tuf) * dp := tufutils.CreateTufDeployment(instance, DeploymentName, RBACName, labels) if err = controllerutil.SetControllerReference(instance, dp, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for Deployment: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for Deployment: %w", err)) } if updated, err = i.Ensure(ctx, dp); err != nil { @@ -51,7 +51,7 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tuf) * Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create TUF: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create TUF: %w", err), instance) } if updated { @@ -62,3 +62,11 @@ func (i deployAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tuf) * return i.Continue() } } + +func (i deployAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Tuf) bool { + return false +} + +func (i deployAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Tuf) *action.Result { + return i.Continue() +} diff --git a/internal/controller/tuf/actions/generate_cert.go b/internal/controller/tuf/actions/generate_cert.go index f4668eab9..47e529685 100644 --- a/internal/controller/tuf/actions/generate_cert.go +++ b/internal/controller/tuf/actions/generate_cert.go @@ -138,3 +138,11 @@ func (i resolveKeysAction) discoverSecret(ctx context.Context, namespace string, return nil, errors.New("secret not found") } + +func (i resolveKeysAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Tuf) bool { + return false +} + +func (i resolveKeysAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Tuf) *action.Result { + return i.Continue() +} diff --git a/internal/controller/tuf/actions/ingress.go b/internal/controller/tuf/actions/ingress.go index 635267733..9a1b4b61e 100644 --- a/internal/controller/tuf/actions/ingress.go +++ b/internal/controller/tuf/actions/ingress.go @@ -40,16 +40,16 @@ func (i ingressAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tuf) svc := &v1.Service{} if err := i.Client.Get(ctx, ok, svc); err != nil { - return i.Failed(fmt.Errorf("could not find service for ingress: %w", err)) + return i.Error(fmt.Errorf("could not find service for ingress: %w", err)) } ingress, err := kubernetes.CreateIngress(ctx, i.Client, *svc, instance.Spec.ExternalAccess, PortName, labels) if err != nil { - return i.Failed(fmt.Errorf("could not create ingress object: %w", err)) + return i.Error(fmt.Errorf("could not create ingress object: %w", err)) } if err = controllerutil.SetControllerReference(instance, ingress, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for Ingress: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for Ingress: %w", err)) } if updated, err = i.Ensure(ctx, ingress); err != nil { @@ -59,7 +59,7 @@ func (i ingressAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tuf) Reason: constants.Failure, Message: err.Error(), }) - return i.Failed(fmt.Errorf("could not create Ingress: %w", err)) + return i.Error(fmt.Errorf("could not create Ingress: %w", err)) } if updated { @@ -70,3 +70,11 @@ func (i ingressAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tuf) return i.Continue() } } + +func (i ingressAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Tuf) bool { + return false +} + +func (i ingressAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Tuf) *action.Result { + return i.Continue() +} diff --git a/internal/controller/tuf/actions/initialize.go b/internal/controller/tuf/actions/initialize.go index 301f04c0a..c7cfcc7b9 100644 --- a/internal/controller/tuf/actions/initialize.go +++ b/internal/controller/tuf/actions/initialize.go @@ -40,7 +40,7 @@ func (i initializeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tu labels := constants.LabelsForComponent(ComponentName, instance.Name) ok, err = commonUtils.DeploymentIsRunning(ctx, i.Client, instance.Namespace, labels) if err != nil { - return i.Failed(err) + return i.Error(err) } if !ok { i.Logger.Info("Waiting for deployment") @@ -58,7 +58,7 @@ func (i initializeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tu ingress := &v12.Ingress{} err = i.Client.Get(ctx, types.NamespacedName{Name: ComponentName, Namespace: instance.Namespace}, ingress) if err != nil { - return i.Failed(err) + return i.Error(err) } if len(ingress.Spec.TLS) > 0 { protocol = "https://" @@ -73,3 +73,11 @@ func (i initializeAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tu return i.StatusUpdate(ctx, instance) } + +func (i initializeAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Tuf) bool { + return false +} + +func (i initializeAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Tuf) *action.Result { + return i.Continue() +} diff --git a/internal/controller/tuf/actions/rbac.go b/internal/controller/tuf/actions/rbac.go index 378c09bd3..3e8b77200 100644 --- a/internal/controller/tuf/actions/rbac.go +++ b/internal/controller/tuf/actions/rbac.go @@ -47,7 +47,7 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tuf) *ac } if err = ctrl.SetControllerReference(instance, sa, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controll reference for SA: %w", err)) + return i.Error(fmt.Errorf("could not set controll reference for SA: %w", err)) } // don't re-enqueue for RBAC in any case (except failure) _, err = i.Ensure(ctx, sa) @@ -58,7 +58,7 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tuf) *ac Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create SA: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create SA: %w", err), instance) } role := kubernetes.CreateRole(instance.Namespace, RBACName, labels, []rbacv1.PolicyRule{ { @@ -74,7 +74,7 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tuf) *ac }) if err = ctrl.SetControllerReference(instance, role, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controll reference for role: %w", err)) + return i.Error(fmt.Errorf("could not set controll reference for role: %w", err)) } _, err = i.Ensure(ctx, role) if err != nil { @@ -84,7 +84,7 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tuf) *ac Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create Role: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create Role: %w", err), instance) } rb := kubernetes.CreateRoleBinding(instance.Namespace, RBACName, labels, rbacv1.RoleRef{ APIGroup: v1.SchemeGroupVersion.Group, @@ -96,7 +96,7 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tuf) *ac }) if err = ctrl.SetControllerReference(instance, rb, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controll reference for roleBinding: %w", err)) + return i.Error(fmt.Errorf("could not set controll reference for roleBinding: %w", err)) } _, err = i.Ensure(ctx, rb) if err != nil { @@ -106,7 +106,15 @@ func (i rbacAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tuf) *ac Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create RoleBinding: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create RoleBinding: %w", err), instance) } return i.Continue() } + +func (i rbacAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Tuf) bool { + return false +} + +func (i rbacAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Tuf) *action.Result { + return i.Continue() +} diff --git a/internal/controller/tuf/actions/servise.go b/internal/controller/tuf/actions/servise.go index 5276116d9..eba09a71a 100644 --- a/internal/controller/tuf/actions/servise.go +++ b/internal/controller/tuf/actions/servise.go @@ -42,7 +42,7 @@ func (i serviceAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tuf) //patch the pregenerated service svc.Spec.Ports[0].Port = instance.Spec.Port if err = controllerutil.SetControllerReference(instance, svc, i.Client.Scheme()); err != nil { - return i.Failed(fmt.Errorf("could not set controller reference for Service: %w", err)) + return i.Error(fmt.Errorf("could not set controller reference for Service: %w", err)) } if updated, err = i.Ensure(ctx, svc); err != nil { meta.SetStatusCondition(&instance.Status.Conditions, metav1.Condition{ @@ -51,7 +51,7 @@ func (i serviceAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tuf) Reason: constants.Failure, Message: err.Error(), }) - return i.FailedWithStatusUpdate(ctx, fmt.Errorf("could not create service: %w", err), instance) + return i.ErrorWithStatusUpdate(ctx, fmt.Errorf("could not create service: %w", err), instance) } if updated { @@ -61,5 +61,12 @@ func (i serviceAction) Handle(ctx context.Context, instance *rhtasv1alpha1.Tuf) } else { return i.Continue() } +} + +func (i serviceAction) CanHandleError(_ context.Context, _ *rhtasv1alpha1.Tuf) bool { + return false +} +func (i serviceAction) HandleError(_ context.Context, _ *rhtasv1alpha1.Tuf) *action.Result { + return i.Continue() } diff --git a/internal/testing/action/result.go b/internal/testing/action/result.go index d22f40536..c2fc9a874 100644 --- a/internal/testing/action/result.go +++ b/internal/testing/action/result.go @@ -1,9 +1,10 @@ package action import ( + "time" + "github.com/securesign/operator/internal/controller/common/action" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "time" ) func Continue() *action.Result { @@ -21,8 +22,12 @@ func Failed(err error) *action.Result { } } -func FailedWithStatusUpdate(err error) *action.Result { - return &action.Result{Result: reconcile.Result{Requeue: false}, Err: err} +func ErrorWithStatusUpdate(_ error) *action.Result { + return &action.Result{Result: reconcile.Result{RequeueAfter: 10 * time.Second}} +} + +func FailWithStatusUpdate(_ error) *action.Result { + return &action.Result{Result: reconcile.Result{Requeue: false}} } func Return() *action.Result { diff --git a/test/e2e/support/common.go b/test/e2e/support/common.go index 72ee1055b..5fcbb5bab 100644 --- a/test/e2e/support/common.go +++ b/test/e2e/support/common.go @@ -4,8 +4,6 @@ import ( "context" "fmt" "io" - v12 "k8s.io/api/apps/v1" - v13 "k8s.io/api/batch/v1" "log" "os" "path/filepath" @@ -14,6 +12,9 @@ import ( "strings" "time" + v12 "k8s.io/api/apps/v1" + v13 "k8s.io/api/batch/v1" + "github.com/docker/docker/api/types" docker "github.com/docker/docker/client" "github.com/google/uuid" @@ -99,6 +100,7 @@ func DumpNamespace(ctx context.Context, cli client.Client, ns string) { "pod.yaml": &v1.PodList{}, "configmap.yaml": &v1.ConfigMapList{}, "deployment.yaml": &v12.DeploymentList{}, + "service.yaml": &v1.ServiceList{}, "job.yaml": &v13.JobList{}, "cronjob.yaml": &v13.CronJobList{}, "event.yaml": &v1.EventList{},