protect against accidental deletes of installed clusters

The admission webhook for CluterDeployments has been modified to inspect
DELETE requests as well. The webhook will reject any DELETE request for
a ClusterDeployment that has the "hive.openshift.io/protected-delete" annotation
set to a true value. In order to delete such a ClusterDeployment, a user
is required to delete the annotation prior to making the DELETE request.

The "deleteProtection" field has been added to the HiveConfig. When the
field is set to "enabled", the clusterdeployment controller will add the
"hive.openshift.io/protected-delete" annotation to the ClusterDeployment
when transitioning the ClusterDeployment to installed.

https://issues.redhat.com/browse/CO-277

Author: staebler

config/crds/hive.openshift.io_hiveconfigs.yaml

@@ -70,6 +70,15 @@ spec:
                       type: boolean
                   type: object
               type: object
+            deleteProtection:
+              description: DeleteProtection can be set to "enabled" to turn on automatic
+                delete protection for ClusterDeployments. When enabled, Hive will
+                add the "hive.openshift.io/protected-delete" annotation to new ClusterDeployments.
+                Once a ClusterDeployment has been installed, a user must remove the
+                annotation from a ClusterDeployment prior to deleting it.
+              enum:
+              - enabled
+              type: string
             deprovisionsDisabled:
               description: DeprovisionsDisabled can be set to true to block deprovision
                 jobs from running.

config/hiveadmission/clusterdeployment-webhook.yaml

@@ -15,6 +15,7 @@ webhooks:
   - operations:
     - CREATE
     - UPDATE
+    - DELETE
     apiGroups:
     - hive.openshift.io
     apiVersions:

pkg/apis/hive/v1/hiveconfig_types.go

@@ -66,6 +66,14 @@ type HiveConfigSpec struct {
 
 	// DeprovisionsDisabled can be set to true to block deprovision jobs from running.
 	DeprovisionsDisabled *bool `json:"deprovisionsDisabled,omitempty"`
+
+	// DeleteProtection can be set to "enabled" to turn on automatic delete protection for ClusterDeployments. When
+	// enabled, Hive will add the "hive.openshift.io/protected-delete" annotation to new ClusterDeployments. Once a
+	// ClusterDeployment has been installed, a user must remove the annotation from a ClusterDeployment prior to
+	// deleting it.
+	// +kubebuilder:validation:Enum=enabled
+	// +optional
+	DeleteProtection DeleteProtectionType `json:"deleteProtection,omitempty"`
 }
 
 // HiveConfigStatus defines the observed state of Hive
@@ -161,6 +169,12 @@ type ManageDNSGCPConfig struct {
 	CredentialsSecretRef corev1.LocalObjectReference `json:"credentialsSecretRef,omitempty"`
 }
 
+type DeleteProtectionType string
+
+const (
+	DeleteProtectionEnabled DeleteProtectionType = "enabled"
+)
+
 // +genclient:nonNamespaced
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/apis/hive/v1/validating-webhooks/clusterdeployment_validating_admission_hook.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"reflect"
 	"regexp"
+	"strconv"
 	"strings"
 
 	log "github.com/sirupsen/logrus"
@@ -21,6 +22,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 
 	hivev1 "github.com/openshift/hive/pkg/apis/hive/v1"
+	"github.com/openshift/hive/pkg/constants"
 	"github.com/openshift/hive/pkg/manageddns"
 )
 
@@ -113,18 +115,18 @@ func (a *ClusterDeploymentValidatingAdmissionHook) Validate(admissionSpec *admis
 
 	contextLogger.Info("Validating request")
 
-	if admissionSpec.Operation == admissionv1beta1.Create {
+	switch admissionSpec.Operation {
+	case admissionv1beta1.Create:
 		return a.validateCreate(admissionSpec)
-	}
-
-	if admissionSpec.Operation == admissionv1beta1.Update {
+	case admissionv1beta1.Update:
 		return a.validateUpdate(admissionSpec)
-	}
-
-	// We're only validating creates and updates at this time, so all other operations are explicitly allowed.
-	contextLogger.Info("Successful validation")
-	return &admissionv1beta1.AdmissionResponse{
-		Allowed: true,
+	case admissionv1beta1.Delete:
+		return a.validateDelete(admissionSpec)
+	default:
+		contextLogger.Info("Successful validation")
+		return &admissionv1beta1.AdmissionResponse{
+			Allowed: true,
+		}
 	}
 }
 
@@ -438,6 +440,59 @@ func (a *ClusterDeploymentValidatingAdmissionHook) validateUpdate(admissionSpec
 	}
 }
 
+// validateDelete specifically validates delete operations for ClusterDeployment objects.
+func (a *ClusterDeploymentValidatingAdmissionHook) validateDelete(request *admissionv1beta1.AdmissionRequest) *admissionv1beta1.AdmissionResponse {
+	logger := log.WithFields(log.Fields{
+		"operation": request.Operation,
+		"group":     request.Resource.Group,
+		"version":   request.Resource.Version,
+		"resource":  request.Resource.Resource,
+		"method":    "validateDelete",
+	})
+
+	oldObject := &hivev1.ClusterDeployment{}
+	if err := a.decoder.DecodeRaw(request.OldObject, oldObject); err != nil {
+		logger.Errorf("Failed unmarshaling Object: %v", err.Error())
+		return &admissionv1beta1.AdmissionResponse{
+			Allowed: false,
+			Result: &metav1.Status{
+				Status: metav1.StatusFailure, Code: http.StatusBadRequest, Reason: metav1.StatusReasonBadRequest,
+				Message: err.Error(),
+			},
+		}
+	}
+
+	logger.Data["object.Name"] = oldObject.Name
+
+	var allErrs field.ErrorList
+
+	if value, present := oldObject.Annotations[constants.ProtectedDeleteAnnotation]; present {
+		if enabled, err := strconv.ParseBool(value); enabled && err == nil {
+			allErrs = append(allErrs, field.Invalid(
+				field.NewPath("metadata", "annotations", constants.ProtectedDeleteAnnotation),
+				oldObject.Annotations[constants.ProtectedDeleteAnnotation],
+				"cannot delete while annotation is present",
+			))
+		} else {
+			logger.WithField(constants.ProtectedDeleteAnnotation, value).Info("Protected Delete annotation present but not set to true")
+		}
+	}
+
+	if len(allErrs) > 0 {
+		logger.WithError(allErrs.ToAggregate()).Info("failed validation")
+		status := errors.NewInvalid(schemaGVK(request.Kind).GroupKind(), request.Name, allErrs).Status()
+		return &admissionv1beta1.AdmissionResponse{
+			Allowed: false,
+			Result:  &status,
+		}
+	}
+
+	logger.Info("Successful validation")
+	return &admissionv1beta1.AdmissionResponse{
+		Allowed: true,
+	}
+}
+
 // isFieldMutable says whether the ClusterDeployment.spec field is meant to be mutable or not.
 func isFieldMutable(value string) bool {
 	for _, mutableField := range mutableFields {

pkg/apis/hive/v1/validating-webhooks/clusterdeployment_validating_admission_hook_test.go

@@ -658,6 +658,38 @@ func TestClusterDeploymentValidate(t *testing.T) {
 			operation:       admissionv1beta1.Create,
 			expectedAllowed: true,
 		},
+		{
+			name:            "Test valid delete",
+			oldObject:       validAWSClusterDeployment(),
+			operation:       admissionv1beta1.Delete,
+			expectedAllowed: true,
+		},
+		{
+			name: "Test protected delete",
+			oldObject: func() *hivev1.ClusterDeployment {
+				cd := validAWSClusterDeployment()
+				if cd.Annotations == nil {
+					cd.Annotations = make(map[string]string, 1)
+				}
+				cd.Annotations[constants.ProtectedDeleteAnnotation] = "true"
+				return cd
+			}(),
+			operation:       admissionv1beta1.Delete,
+			expectedAllowed: false,
+		},
+		{
+			name: "Test protected delete annotation false",
+			oldObject: func() *hivev1.ClusterDeployment {
+				cd := validAWSClusterDeployment()
+				if cd.Annotations == nil {
+					cd.Annotations = make(map[string]string, 1)
+				}
+				cd.Annotations[constants.ProtectedDeleteAnnotation] = "false"
+				return cd
+			}(),
+			operation:       admissionv1beta1.Delete,
+			expectedAllowed: true,
+		},
 	}
 
 	for _, tc := range cases {

pkg/constants/constants.go

@@ -128,6 +128,14 @@ const (
 	// for the cluster provision to complete by running `openshift-install wait-for install-complete` command.
 	WaitForInstallCompleteExecutionsAnnotation = "hive.openshift.io/wait-for-install-complete-executions"
 
+	// ProtectedDeleteAnnotation is an annotation used on ClusterDeployments to indicate that the ClusterDeployment
+	// cannot be deleted. The annotation must be removed in order to delete the ClusterDeployment.
+	ProtectedDeleteAnnotation = "hive.openshift.io/protected-delete"
+
+	// ProtectedDeleteEnvVar is the name of the environment variable used to tell the controller manager whether
+	// protected delete is enabled.
+	ProtectedDeleteEnvVar = "PROTECTED_DELETE"
+
 	// ManagedDomainsFileEnvVar if present, points to a simple text
 	// file that includes a valid managed domain per line. Cluster deployments
 	// requesting that their domains be managed must have a base domain

pkg/controller/clusterdeployment/clusterdeployment_controller.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"reflect"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -155,6 +156,13 @@ func NewReconciler(mgr manager.Manager) reconcile.Reconciler {
 	r.remoteClusterAPIClientBuilder = func(cd *hivev1.ClusterDeployment) remoteclient.Builder {
 		return remoteclient.NewBuilder(r.Client, cd, controllerName)
 	}
+
+	protectedDeleteEnvVar := os.Getenv(constants.ProtectedDeleteEnvVar)
+	if protectedDelete, err := strconv.ParseBool(protectedDeleteEnvVar); protectedDelete && err == nil {
+		logger.Info("Protected Delete enabled")
+		r.protectedDelete = true
+	}
+
 	return r
 }
 
@@ -248,6 +256,8 @@ type ReconcileClusterDeployment struct {
 	// remoteClusterAPIClientBuilder is a function pointer to the function that gets a builder for building a client
 	// for the remote cluster's API server
 	remoteClusterAPIClientBuilder func(cd *hivev1.ClusterDeployment) remoteclient.Builder
+
+	protectedDelete bool
 }
 
 // Reconcile reads that state of the cluster for a ClusterDeployment object and makes changes based on the state read
@@ -808,6 +818,13 @@ func (r *ReconcileClusterDeployment) reconcileCompletedProvision(cd *hivev1.Clus
 
 	cd.Spec.Installed = true
 
+	if r.protectedDelete {
+		if _, annotationPresent := cd.Annotations[constants.ProtectedDeleteAnnotation]; !annotationPresent {
+			initializeAnnotations(cd)
+			cd.Annotations[constants.ProtectedDeleteAnnotation] = "true"
+		}
+	}
+
 	if err := r.Update(context.TODO(), cd); err != nil {
 		cdLog.WithError(err).Log(controllerutils.LogLevel(err), "failed to set the Installed flag")
 		return reconcile.Result{}, err

pkg/controller/clusterdeployment/clusterdeployment_controller_test.go

@@ -122,6 +122,7 @@ func TestClusterDeploymentReconcile(t *testing.T) {
 		expectPendingCreation   bool
 		expectConsoleRouteFetch bool
 		validate                func(client.Client, *testing.T)
+		reconcilerSetup         func(*ReconcileClusterDeployment)
 	}{
 		{
 			name: "Add finalizer",
@@ -284,6 +285,28 @@ func TestClusterDeploymentReconcile(t *testing.T) {
 				cd := getCD(c)
 				if assert.NotNil(t, cd, "missing clusterdeployment") {
 					assert.True(t, cd.Spec.Installed, "expected cluster to be installed")
+					assert.NotContains(t, cd.Annotations, constants.ProtectedDeleteAnnotation, "unexpected protected delete annotation")
+				}
+			},
+		},
+		{
+			name: "Completed provision with protected delete",
+			existing: []runtime.Object{
+				testClusterDeploymentWithProvision(),
+				testSuccessfulProvision(),
+				testMetadataConfigMap(),
+				testSecret(corev1.SecretTypeOpaque, adminKubeconfigSecret, "kubeconfig", adminKubeconfig),
+				testSecret(corev1.SecretTypeDockerConfigJson, pullSecretSecret, corev1.DockerConfigJsonKey, "{}"),
+				testSecret(corev1.SecretTypeDockerConfigJson, constants.GetMergedPullSecretName(testClusterDeployment()), corev1.DockerConfigJsonKey, "{}"),
+			},
+			reconcilerSetup: func(r *ReconcileClusterDeployment) {
+				r.protectedDelete = true
+			},
+			validate: func(c client.Client, t *testing.T) {
+				cd := getCD(c)
+				if assert.NotNil(t, cd, "missing clusterdeployment") {
+					assert.True(t, cd.Spec.Installed, "expected cluster to be installed")
+					assert.Equal(t, "true", cd.Annotations[constants.ProtectedDeleteAnnotation], "unexpected protected delete annotation")
 				}
 			},
 		},
@@ -1129,6 +1152,10 @@ func TestClusterDeploymentReconcile(t *testing.T) {
 				remoteClusterAPIClientBuilder: func(*hivev1.ClusterDeployment) remoteclient.Builder { return mockRemoteClientBuilder },
 			}
 
+			if test.reconcilerSetup != nil {
+				test.reconcilerSetup(rcd)
+			}
+
 			reconcileRequest := reconcile.Request{
 				NamespacedName: types.NamespacedName{
 					Name:      testName,

pkg/operator/assets/bindata.go

@@ -133,6 +133,14 @@ func (r *ReconcileHiveConfig) deployHive(hLog log.FieldLogger, h *resource.Helpe
 		hiveContainer.Env = append(hiveContainer.Env, tmpEnvVar)
 	}
 
+	if instance.Spec.DeleteProtection == hivev1.DeleteProtectionEnabled {
+		hLog.Info("Delete Protection enabled")
+		hiveContainer.Env = append(hiveContainer.Env, corev1.EnvVar{
+			Name:  hiveconstants.ProtectedDeleteEnvVar,
+			Value: "true",
+		})
+	}
+
 	if err := r.includeAdditionalCAs(hLog, h, instance, hiveDeployment); err != nil {
 		return err
 	}