diff --git a/pkg/controller/statefulset/stateful_pod_control.go b/pkg/controller/statefulset/stateful_pod_control.go index b995d0892f5b2..a67ef41ff8343 100644 --- a/pkg/controller/statefulset/stateful_pod_control.go +++ b/pkg/controller/statefulset/stateful_pod_control.go @@ -50,6 +50,8 @@ type StatefulPodControlInterface interface { // DeleteStatefulPod deletes a Pod in a StatefulSet. The pods PVCs are not deleted. If the delete is successful, // the returned error is nil. DeleteStatefulPod(set *apps.StatefulSet, pod *v1.Pod) error + // CreatePersistentVolumeClaims creates all of the required PersistentVolumeClaims for pod + CreatePersistentVolumeClaims(set *apps.StatefulSet, pod *v1.Pod) error } func NewRealStatefulPodControl( @@ -74,7 +76,7 @@ type realStatefulPodControl struct { func (spc *realStatefulPodControl) CreateStatefulPod(set *apps.StatefulSet, pod *v1.Pod) error { // Create the Pod's PVCs prior to creating the Pod - if err := spc.createPersistentVolumeClaims(set, pod); err != nil { + if err := spc.CreatePersistentVolumeClaims(set, pod); err != nil { spc.recordPodEvent("create", set, pod, err) return err } @@ -103,7 +105,7 @@ func (spc *realStatefulPodControl) UpdateStatefulPod(set *apps.StatefulSet, pod if !storageMatches(set, pod) { updateStorage(set, pod) consistent = false - if err := spc.createPersistentVolumeClaims(set, pod); err != nil { + if err := spc.CreatePersistentVolumeClaims(set, pod); err != nil { spc.recordPodEvent("update", set, pod, err) return err } @@ -174,11 +176,11 @@ func (spc *realStatefulPodControl) recordClaimEvent(verb string, set *apps.State } } -// createPersistentVolumeClaims creates all of the required PersistentVolumeClaims for pod, which must be a member of +// CreatePersistentVolumeClaims creates all of the required PersistentVolumeClaims for pod, which must be a member of // set. If all of the claims for Pod are successfully created, the returned error is nil. If creation fails, this method // may be called again until no error is returned, indicating the PersistentVolumeClaims for pod are consistent with // set's Spec. -func (spc *realStatefulPodControl) createPersistentVolumeClaims(set *apps.StatefulSet, pod *v1.Pod) error { +func (spc *realStatefulPodControl) CreatePersistentVolumeClaims(set *apps.StatefulSet, pod *v1.Pod) error { var errs []error for _, claim := range getPersistentVolumeClaims(set, pod) { pvc, err := spc.pvcLister.PersistentVolumeClaims(claim.Namespace).Get(claim.Name) diff --git a/pkg/controller/statefulset/stateful_set_control.go b/pkg/controller/statefulset/stateful_set_control.go index 8e71c53e4555a..f74bc7613b5c6 100644 --- a/pkg/controller/statefulset/stateful_set_control.go +++ b/pkg/controller/statefulset/stateful_set_control.go @@ -426,6 +426,18 @@ func (ssc *defaultStatefulSetControl) updateStatefulSet( // pod created, no more work possible for this round continue } + + // If the Pod is in pending state then trigger PVC creation to create missing PVCs + if isPending(replicas[i]) { + klog.V(4).Infof( + "StatefulSet %s/%s is triggering PVC creation for pending Pod %s", + set.Namespace, + set.Name, + replicas[i].Name) + if err := ssc.podControl.CreatePersistentVolumeClaims(set, replicas[i]); err != nil { + return &status, err + } + } // If we find a Pod that is currently terminating, we must wait until graceful deletion // completes before we continue to make progress. if isTerminating(replicas[i]) && monotonic { diff --git a/pkg/controller/statefulset/stateful_set_control_test.go b/pkg/controller/statefulset/stateful_set_control_test.go index adb6a82b1a6d1..4ed20bc665227 100644 --- a/pkg/controller/statefulset/stateful_set_control_test.go +++ b/pkg/controller/statefulset/stateful_set_control_test.go @@ -90,6 +90,7 @@ func TestStatefulSetControl(t *testing.T) { {UpdatePodFailure, simpleSetFn}, {UpdateSetStatusFailure, simpleSetFn}, {PodRecreateDeleteFailure, simpleSetFn}, + {RecreatesPVCForPendingPod, simpleSetFn}, } for _, testCase := range testCases { @@ -443,6 +444,46 @@ func PodRecreateDeleteFailure(t *testing.T, set *apps.StatefulSet, invariants in } } +func RecreatesPVCForPendingPod(t *testing.T, set *apps.StatefulSet, invariants invariantFunc) { + client := fake.NewSimpleClientset() + spc, _, ssc, stop := setupController(client) + defer close(stop) + selector, err := metav1.LabelSelectorAsSelector(set.Spec.Selector) + if err != nil { + t.Error(err) + } + pods, err := spc.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Error(err) + } + if err := ssc.UpdateStatefulSet(set, pods); err != nil { + t.Errorf("Error updating StatefulSet %s", err) + } + if err := invariants(set, spc); err != nil { + t.Error(err) + } + pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Error(err) + } + for _, claim := range getPersistentVolumeClaims(set, pods[0]) { + spc.claimsIndexer.Delete(&claim) + } + pods[0].Status.Phase = v1.PodPending + spc.podsIndexer.Update(pods[0]) + if err := ssc.UpdateStatefulSet(set, pods); err != nil { + t.Errorf("Error updating StatefulSet %s", err) + } + // invariants check if there any missing PVCs for the Pods + if err := invariants(set, spc); err != nil { + t.Error(err) + } + pods, err = spc.podsLister.Pods(set.Namespace).List(selector) + if err != nil { + t.Error(err) + } +} + func TestStatefulSetControlScaleDownDeleteError(t *testing.T) { invariants := assertMonotonicInvariants set := newStatefulSet(3) @@ -1767,6 +1808,13 @@ func (spc *fakeStatefulPodControl) DeleteStatefulPod(set *apps.StatefulSet, pod return nil } +func (spc *fakeStatefulPodControl) CreatePersistentVolumeClaims(set *apps.StatefulSet, pod *v1.Pod) error { + for _, claim := range getPersistentVolumeClaims(set, pod) { + spc.claimsIndexer.Update(&claim) + } + return nil +} + var _ StatefulPodControlInterface = &fakeStatefulPodControl{} type fakeStatefulSetStatusUpdater struct { diff --git a/pkg/controller/statefulset/stateful_set_utils.go b/pkg/controller/statefulset/stateful_set_utils.go index 12600fd632356..6e6925290839e 100644 --- a/pkg/controller/statefulset/stateful_set_utils.go +++ b/pkg/controller/statefulset/stateful_set_utils.go @@ -24,7 +24,7 @@ import ( "strconv" apps "k8s.io/api/apps/v1" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/strategicpatch" @@ -77,7 +77,7 @@ func getParentName(pod *v1.Pod) string { return parent } -// getOrdinal gets pod's ordinal. If pod has no ordinal, -1 is returned. +// getOrdinal gets pod's ordinal. If pod has no ordinal, -1 is returned. func getOrdinal(pod *v1.Pod) int { _, ordinal := getParentNameAndOrdinal(pod) return ordinal @@ -209,6 +209,11 @@ func isCreated(pod *v1.Pod) bool { return pod.Status.Phase != "" } +// isPending returns true if pod has a Phase of PodPending +func isPending(pod *v1.Pod) bool { + return pod.Status.Phase == v1.PodPending +} + // isFailed returns true if pod has a Phase of PodFailed func isFailed(pod *v1.Pod) bool { return pod.Status.Phase == v1.PodFailed