Merge pull request #142956 from galal-hussein/kep_3541_beta

KEP-3541 beta (StatefulSetRecreateStrategy)

pkg/apis/apps/types.go

@@ -93,7 +93,7 @@ const (
 	// RecreateStatefulSetStrategyType indicates that all existing pods will be
 	// deleted and fully terminated before any new-revision pods are created.
 	// This ensures that old and new revision Pods never run at the same time.
-	// This is an alpha type and requires enabling StatefulSetRecreateStrategy feature gate.
+	// This is a beta type and requires enabling StatefulSetRecreateStrategy feature gate.
 	// Switching to the Recreate strategy is allowed when the feature gate is enabled,
 	// and switching away from it to a different update strategy is also allowed.
 	//
@@ -307,7 +307,7 @@ type StatefulSetConditionType string
 
 const (
 	// StatefulSetProgressing means the StatefulSet is progressing through an update.
-	// This condition is an alpha type and requires enabling StatefulSetRecreateStrategy feature gate.
+	// This condition is a beta type and requires enabling StatefulSetRecreateStrategy feature gate.
 	StatefulSetProgressing StatefulSetConditionType = "Progressing"
 )
 

pkg/features/kube_features.go

@@ -2013,6 +2013,7 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
 
 	StatefulSetRecreateStrategy: {
 		{Version: version.MustParse("1.37"), Default: false, PreRelease: featuregate.Alpha},
+		{Version: version.MustParse("1.38"), Default: true, PreRelease: featuregate.Beta},
 	},
 
 	StatefulSetSemanticRevisionComparison: {

pkg/generated/openapi/zz_generated.openapi.go

@@ -9059,7 +9059,7 @@ func schema_k8sio_api_apps_v1_StatefulSetUpdateStrategy(ref common.ReferenceCall
 				Properties: map[string]spec.Schema{
 					"type": {
 						SchemaProps: spec.SchemaProps{
-							Description: "type indicates the type of the StatefulSetUpdateStrategy. Default is RollingUpdate.\n\nPossible enum values:\n - `\"OnDelete\"` disables ordered rolling restarts. Version tracking is done on a best-effort basis - the controller will try to eventually converge StatefulSet's currentRevision with updateRevision. Pods are recreated from the StatefulSetSpec when they are manually deleted. When a scale operation is performed with this strategy, new Pods will be created from the specification version indicated by the StatefulSet's updateRevision.\n - `\"Recreate\"` indicates that all existing pods will be deleted and fully terminated before any new-revision pods are created. This ensures that old and new revision Pods never run at the same time. This is an alpha type and requires enabling StatefulSetRecreateStrategy feature gate. Switching to the Recreate strategy is allowed when the feature gate is enabled, and switching away from it to a different update strategy is also allowed. +featureGate=StatefulSetRecreateStrategy\n - `\"RollingUpdate\"` indicates that update will be applied to all Pods in the StatefulSet with respect to the StatefulSet ordering constraints. When a scale operation is performed with this strategy, new Pods will be created from the specification version indicated by the StatefulSet's updateRevision.",
+							Description: "type indicates the type of the StatefulSetUpdateStrategy. Default is RollingUpdate.\n\nPossible enum values:\n - `\"OnDelete\"` disables ordered rolling restarts. Version tracking is done on a best-effort basis - the controller will try to eventually converge StatefulSet's currentRevision with updateRevision. Pods are recreated from the StatefulSetSpec when they are manually deleted. When a scale operation is performed with this strategy, new Pods will be created from the specification version indicated by the StatefulSet's updateRevision.\n - `\"Recreate\"` indicates that all existing pods will be deleted and fully terminated before any new-revision pods are created. This ensures that old and new revision Pods never run at the same time. This is a beta type and requires enabling StatefulSetRecreateStrategy feature gate. Switching to the Recreate strategy is allowed when the feature gate is enabled, and switching away from it to a different update strategy is also allowed. +featureGate=StatefulSetRecreateStrategy\n - `\"RollingUpdate\"` indicates that update will be applied to all Pods in the StatefulSet with respect to the StatefulSet ordering constraints. When a scale operation is performed with this strategy, new Pods will be created from the specification version indicated by the StatefulSet's updateRevision.",
 							Type:        []string{"string"},
 							Format:      "",
 							Enum:        []interface{}{"OnDelete", "Recreate", "RollingUpdate"},

staging/src/k8s.io/api/apps/v1/types.go

@@ -118,7 +118,7 @@ const (
 	// RecreateStatefulSetStrategyType indicates that all existing pods will be
 	// deleted and fully terminated before any new-revision pods are created.
 	// This ensures that old and new revision Pods never run at the same time.
-	// This is an alpha type and requires enabling StatefulSetRecreateStrategy feature gate.
+	// This is a beta type and requires enabling StatefulSetRecreateStrategy feature gate.
 	// Switching to the Recreate strategy is allowed when the feature gate is enabled,
 	// and switching away from it to a different update strategy is also allowed.
 	//
@@ -354,7 +354,7 @@ type StatefulSetConditionType string
 
 const (
 	// StatefulSetProgressing means the StatefulSet is progressing through an update.
-	// This condition is an alpha type and requires enabling StatefulSetRecreateStrategy feature gate.
+	// This condition is a beta type and requires enabling StatefulSetRecreateStrategy feature gate.
 	StatefulSetProgressing StatefulSetConditionType = "Progressing"
 )
 

test/compatibility_lifecycle/reference/feature_list.md

@@ -208,7 +208,7 @@
 | StaleControllerConsistencyJob | :ballot_box_with_check: 1.36+ |  |  | 1.36– |  |  | AtomicFIFO | [code](https://cs.k8s.io/?q=%5CbStaleControllerConsistencyJob%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbStaleControllerConsistencyJob%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) |
 | StaleControllerConsistencyReplicaSet | :ballot_box_with_check: 1.36+ |  |  | 1.36– |  |  | AtomicFIFO | [code](https://cs.k8s.io/?q=%5CbStaleControllerConsistencyReplicaSet%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbStaleControllerConsistencyReplicaSet%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) |
 | StaleControllerConsistencyStatefulSet | :ballot_box_with_check: 1.36+ |  |  | 1.36– |  |  | AtomicFIFO | [code](https://cs.k8s.io/?q=%5CbStaleControllerConsistencyStatefulSet%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbStaleControllerConsistencyStatefulSet%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) |
-| StatefulSetRecreateStrategy |  |  | 1.37– |  |  |  |  | [code](https://cs.k8s.io/?q=%5CbStatefulSetRecreateStrategy%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbStatefulSetRecreateStrategy%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) |
+| StatefulSetRecreateStrategy | :ballot_box_with_check: 1.38+ |  | 1.37 | 1.38– |  |  |  | [code](https://cs.k8s.io/?q=%5CbStatefulSetRecreateStrategy%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbStatefulSetRecreateStrategy%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) |
 | StatefulSetSemanticRevisionComparison | :ballot_box_with_check: 1.0+ |  |  | 1.0– |  |  |  | [code](https://cs.k8s.io/?q=%5CbStatefulSetSemanticRevisionComparison%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbStatefulSetSemanticRevisionComparison%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) |
 | StorageCapacityScoring | :ballot_box_with_check: 1.37+ |  | 1.33–1.36 | 1.37– |  |  |  | [code](https://cs.k8s.io/?q=%5CbStorageCapacityScoring%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbStorageCapacityScoring%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) |
 | StorageNamespaceIndex | :ballot_box_with_check: 1.30+ |  |  | 1.30–1.32 |  | 1.33– |  | [code](https://cs.k8s.io/?q=%5CbStorageNamespaceIndex%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbStorageNamespaceIndex%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) |

test/compatibility_lifecycle/reference/versioned_feature_list.yaml

@@ -1910,6 +1910,10 @@
     lockToDefault: false
     preRelease: Alpha
     version: "1.37"
+  - default: true
+    lockToDefault: false
+    preRelease: Beta
+    version: "1.38"
 - name: StatefulSetSemanticRevisionComparison
   versionedSpecs:
   - default: true

test/integration/statefulset/statefulset_test.go

@@ -31,6 +31,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/json"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/dynamic"
@@ -373,6 +374,28 @@ func setPodsReadyCondition(t *testing.T, clientSet clientset.Interface, pods *v1
 	}
 }
 
+func setPendingPodsReady(t *testing.T, c clientset.Interface, pods *v1.PodList) {
+	pendingPods := v1.PodList{}
+	for _, pod := range pods.Items {
+		if pod.Status.Phase == v1.PodPending {
+			pendingPods.Items = append(pendingPods.Items, pod)
+		}
+	}
+	if len(pendingPods.Items) > 0 {
+		setPodsReadyCondition(t, c, &pendingPods, v1.ConditionTrue, time.Now())
+	}
+}
+
+func hasMixedRevisionPods(pods *v1.PodList) bool {
+	revisions := sets.New[string]()
+	for _, pod := range pods.Items {
+		if rev := pod.Labels[appsv1.ControllerRevisionHashLabelKey]; rev != "" {
+			revisions.Insert(rev)
+		}
+	}
+	return revisions.Len() > 1
+}
+
 // add for issue: https://github.com/kubernetes/kubernetes/issues/108837
 func TestStatefulSetStatusWithPodFail(t *testing.T) {
 	tCtx := ktesting.Init(t)
@@ -839,7 +862,7 @@ func TestRecreateStatefulSetUpdate(t *testing.T) {
 	}
 
 	newImage := "new-image"
-	updateSTS(t, stsClient, sts.Name, func(sts *appsv1.StatefulSet) {
+	sts = updateSTS(t, stsClient, sts.Name, func(sts *appsv1.StatefulSet) {
 		sts.Spec.Template.Spec.Containers[0].Image = newImage
 	})
 
@@ -849,30 +872,12 @@ func TestRecreateStatefulSetUpdate(t *testing.T) {
 			return false, err
 		}
 		currentPods := getPods(t, podClient, labelMap)
-
-		// At no point mixed revisions should exists between pods in recreate
-		revisions := make(map[string]bool)
-		for _, pod := range currentPods.Items {
-			rev := pod.Labels[appsv1.ControllerRevisionHashLabelKey]
-			if rev != "" {
-				revisions[rev] = true
-			}
+		if hasMixedRevisionPods(currentPods) {
+			return false, fmt.Errorf("found mixed revisions among pods with Recreate strategy")
 		}
-		if len(revisions) > 1 {
-			return false, fmt.Errorf("found mixed revisions among pods: %v", revisions)
-		}
-
-		pendingPods := v1.PodList{}
-		for _, pod := range currentPods.Items {
-			if pod.Status.Phase == v1.PodPending {
-				pendingPods.Items = append(pendingPods.Items, pod)
-			}
-		}
-		if len(pendingPods.Items) > 0 {
-			setPodsReadyCondition(t, c, &pendingPods, v1.ConditionTrue, time.Now())
-		}
-
-		return ss.Status.ReadyReplicas == *ss.Spec.Replicas &&
+		setPendingPodsReady(t, c, currentPods)
+		return ss.Status.ObservedGeneration == ss.Generation &&
+			ss.Status.ReadyReplicas == *ss.Spec.Replicas &&
 			ss.Status.CurrentRevision == ss.Status.UpdateRevision, nil
 	}); err != nil {
 		t.Fatalf("Timed out waiting for recreate update to complete: %v", err)
@@ -917,3 +922,138 @@ func TestRecreateStatefulSetUpdate(t *testing.T) {
 		}
 	}
 }
+
+func TestStatefulSetUpdateStrategySwitch(t *testing.T) {
+	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetRecreateStrategy, true)
+	tCtx, rm, informers, c := scSetup(t)
+	cancel := runControllerAndInformers(tCtx, rm, informers)
+	defer cancel()
+
+	tests := []struct {
+		name           string
+		namespace      string
+		initStrategy   appsv1.StatefulSetUpdateStrategy
+		switchStrategy appsv1.StatefulSetUpdateStrategy
+	}{
+		{
+			name:           "RollingUpdate to Recreate",
+			namespace:      "rolling-to-recreate",
+			initStrategy:   appsv1.StatefulSetUpdateStrategy{Type: appsv1.RollingUpdateStatefulSetStrategyType},
+			switchStrategy: appsv1.StatefulSetUpdateStrategy{Type: appsv1.RecreateStatefulSetStrategyType},
+		},
+		{
+			name:           "OnDelete to Recreate",
+			namespace:      "ondelete-to-recreate",
+			initStrategy:   appsv1.StatefulSetUpdateStrategy{Type: appsv1.OnDeleteStatefulSetStrategyType},
+			switchStrategy: appsv1.StatefulSetUpdateStrategy{Type: appsv1.RecreateStatefulSetStrategyType},
+		},
+		{
+			name:           "Recreate to RollingUpdate",
+			namespace:      "recreate-to-rolling",
+			initStrategy:   appsv1.StatefulSetUpdateStrategy{Type: appsv1.RecreateStatefulSetStrategyType},
+			switchStrategy: appsv1.StatefulSetUpdateStrategy{Type: appsv1.RollingUpdateStatefulSetStrategyType},
+		},
+		{
+			name:           "Recreate to OnDelete",
+			namespace:      "recreate-to-ondelete",
+			initStrategy:   appsv1.StatefulSetUpdateStrategy{Type: appsv1.RecreateStatefulSetStrategyType},
+			switchStrategy: appsv1.StatefulSetUpdateStrategy{Type: appsv1.OnDeleteStatefulSetStrategyType},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ns := framework.CreateNamespaceOrDie(c, test.namespace, t)
+			defer framework.DeleteNamespaceOrDie(c, ns, t)
+
+			createHeadlessService(t, c, newHeadlessService(ns.Name))
+			labelMap := labelMap()
+			sts := newSTS("sts", ns.Name, 3)
+			sts.Spec.UpdateStrategy = test.initStrategy
+			stss, _ := createSTSsPods(t, c, []*appsv1.StatefulSet{sts}, []*v1.Pod{})
+			sts = stss[0]
+			waitSTSStable(t, c, sts)
+
+			podClient := c.CoreV1().Pods(ns.Name)
+			pods := getPods(t, podClient, labelMap)
+			if len(pods.Items) != 3 {
+				t.Fatalf("len(pods) = %d, want 3", len(pods.Items))
+			}
+			setPodsReadyCondition(t, c, pods, v1.ConditionTrue, time.Now())
+
+			stsClient := c.AppsV1().StatefulSets(ns.Name)
+			sts, err := stsClient.Get(tCtx, sts.Name, metav1.GetOptions{})
+			if err != nil {
+				t.Fatalf("Failed to get StatefulSet: %v", err)
+			}
+			oldCurrentRevision := sts.Status.CurrentRevision
+
+			oldPodUIDs := make(map[string]types.UID)
+			for _, pod := range pods.Items {
+				oldPodUIDs[pod.Name] = pod.UID
+			}
+
+			newImage := "new-image"
+			sts = updateSTS(t, stsClient, sts.Name, func(sts *appsv1.StatefulSet) {
+				sts.Spec.Template.Spec.Containers[0].Image = newImage
+				sts.Spec.UpdateStrategy = test.switchStrategy
+			})
+
+			if test.switchStrategy.Type == appsv1.OnDeleteStatefulSetStrategyType {
+				waitSTSStable(t, c, sts)
+				for _, pod := range getPods(t, podClient, labelMap).Items {
+					if err := podClient.Delete(tCtx, pod.Name, metav1.DeleteOptions{}); err != nil {
+						t.Fatalf("Failed to delete pod %s: %v", pod.Name, err)
+					}
+				}
+			}
+
+			mixedRevisions := false
+			if err := wait.PollUntilContextTimeout(tCtx, pollInterval, pollTimeout, false, func(ctx context.Context) (bool, error) {
+				ss, err := stsClient.Get(ctx, sts.Name, metav1.GetOptions{})
+				if err != nil {
+					return false, err
+				}
+				currentPods := getPods(t, podClient, labelMap)
+				if hasMixedRevisionPods(currentPods) {
+					mixedRevisions = true
+				}
+				setPendingPodsReady(t, c, currentPods)
+				return ss.Status.ObservedGeneration >= ss.Generation &&
+					ss.Status.ReadyReplicas == *ss.Spec.Replicas &&
+					ss.Status.CurrentRevision == ss.Status.UpdateRevision, nil
+			}); err != nil {
+				t.Fatalf("Timed out waiting for update of StatefulSet %s/%s to complete: %v", sts.Namespace, sts.Name, err)
+			}
+
+			if test.switchStrategy.Type == appsv1.RecreateStatefulSetStrategyType && mixedRevisions {
+				t.Fatalf("StatefulSet %s/%s had mixed revision pods with Recreate strategy", sts.Namespace, sts.Name)
+			}
+			if test.switchStrategy.Type == appsv1.RollingUpdateStatefulSetStrategyType && !mixedRevisions {
+				t.Fatalf("StatefulSet %s/%s never had old and new revision pods together with RollingUpdate strategy", sts.Namespace, sts.Name)
+			}
+
+			sts, err = stsClient.Get(tCtx, sts.Name, metav1.GetOptions{})
+			if err != nil {
+				t.Fatalf("Failed to get StatefulSet after update: %v", err)
+			}
+			if sts.Status.CurrentRevision == oldCurrentRevision {
+				t.Fatalf("CurrentRevision should have changed from %s after update", oldCurrentRevision)
+			}
+
+			// Verify all pods were recreated and have the new image.
+			pods = getPods(t, podClient, labelMap)
+			if len(pods.Items) != int(*sts.Spec.Replicas) {
+				t.Fatalf("Expected %d pods, got %d", *sts.Spec.Replicas, len(pods.Items))
+			}
+			for _, pod := range pods.Items {
+				if oldUID, ok := oldPodUIDs[pod.Name]; ok && pod.UID == oldUID {
+					t.Fatalf("Pod %s was not recreated", pod.Name)
+				}
+				if pod.Spec.Containers[0].Image != newImage {
+					t.Fatalf("Pod %s has image %s, want %s", pod.Name, pod.Spec.Containers[0].Image, newImage)
+				}
+			}
+		})
+	}
+}