wait for a given duration in case of imagePullBackOff

We have implemented imagePullBackOff as fail fast. The issue with this approach is, the node where the pod is scheduled often experiences registry rate limit. The image pull failure because of the rate limit returns the same warning (reason: Failed and message: ImagePullBackOff). The pod can potentially recover after waiting for enough time until the cap is expired. Kubernetes can then successfully pull the image and bring the pod up. Introducing a default configuration to specify cluster level timeout to allow the imagePullBackOff to retry for a given duration. Once that duration has passed, return a permanent failure. tektoncd#5987 tektoncd#7184 This is a manual cheery-pick of tektoncd#7666 Signed-off-by: Priti Desai <[email protected]>
pritidesai · Feb 15, 2024 · e2231cf · e2231cf
1 parent c58fa31
commit e2231cf
Show file tree

Hide file tree

Showing 9 changed files with 246 additions and 22 deletions.
diff --git a/config/config-defaults.yaml b/config/config-defaults.yaml
@@ -86,3 +86,9 @@ data:
     # default-resolver-type contains the default resolver type to be used in the cluster,
     # no default-resolver-type is specified by default
     default-resolver-type:
+    
+    # default-imagepullbackoff-timeout contains the default duration to wait
+    # before requeuing the TaskRun to retry, specifying 0 here is equivalent to fail fast
+    # possible values could be 1m, 5m, 10s, 1h, etc
+    # default-imagepullbackoff-timeout: "5m"
+
diff --git a/docs/additional-configs.md b/docs/additional-configs.md
@@ -31,6 +31,7 @@ installation.
     - [Verify the transparency logs using `rekor-cli`](#verify-the-transparency-logs-using-rekor-cli)
   - [Verify Tekton Resources](#verify-tekton-resources)
   - [Pipelinerun with Affinity Assistant](#pipelineruns-with-affinity-assistant)
+  - [TaskRuns with `imagePullBackOff` Timeout](#taskruns-with-imagepullbackoff-timeout)
   - [Next steps](#next-steps)
 
 
@@ -607,6 +608,26 @@ please take a look at [Trusted Resources](./trusted-resources.md).
 The cluster operators can review the [guidelines](developers/affinity-assistant.md) to `cordon` a node in the cluster
 with the tekton controller and the affinity assistant is enabled.
 
+## TaskRuns with `imagePullBackOff` Timeout
+
+Tekton pipelines has adopted a fail fast strategy with a taskRun failing with `TaskRunImagePullFailed` in case of an
+`imagePullBackOff`. This can be limited in some cases, and it generally depends on the infrastructure. To allow the
+cluster operators to decide whether to wait in case of an `imagePullBackOff`, a setting is available to configure
+the wait time in minutes such that the controller will wait for the specified duration before declaring a failure.
+For example, with the following `config-defaults`, the controller does not mark the taskRun as failure for 5 minutes since
+the pod is scheduled in case the image pull fails with `imagePullBackOff`.
+See issue https://github.com/tektoncd/pipeline/issues/5987 for more details.
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config-defaults
+  namespace: tekton-pipelines
+data:
+  default-imagepullbackoff-timeout: "5"
+```
+
 ## Next steps
 
 To get started with Tekton check the [Introductory tutorials][quickstarts],

diff --git a/pkg/apis/config/default.go b/pkg/apis/config/default.go
@@ -46,6 +46,8 @@ const (
 	DefaultMaxMatrixCombinationsCount = 256
 	// DefaultResolverTypeValue is used when no default resolver type is specified
 	DefaultResolverTypeValue = ""
+	// DefaultImagePullBackOffTimeout is used when no imagePullBackOff timeout is specified
+	DefaultImagePullBackOffTimeout = 0 * time.Minute
 
 	defaultTimeoutMinutesKey             = "default-timeout-minutes"
 	defaultServiceAccountKey             = "default-service-account"
@@ -57,6 +59,7 @@ const (
 	defaultMaxMatrixCombinationsCountKey = "default-max-matrix-combinations-count"
 	defaultForbiddenEnv                  = "default-forbidden-env"
 	defaultResolverTypeKey               = "default-resolver-type"
+	defaultImagePullBackOffTimeout       = "default-imagepullbackoff-timeout"
 )
 
 // DefaultConfig holds all the default configurations for the config.
@@ -75,6 +78,7 @@ type Defaults struct {
 	DefaultMaxMatrixCombinationsCount int
 	DefaultForbiddenEnv               []string
 	DefaultResolverType               string
+	DefaultImagePullBackOffTimeout    time.Duration
 }
 
 // GetDefaultsConfigName returns the name of the configmap containing all
@@ -105,6 +109,7 @@ func (cfg *Defaults) Equals(other *Defaults) bool {
 		other.DefaultTaskRunWorkspaceBinding == cfg.DefaultTaskRunWorkspaceBinding &&
 		other.DefaultMaxMatrixCombinationsCount == cfg.DefaultMaxMatrixCombinationsCount &&
 		other.DefaultResolverType == cfg.DefaultResolverType &&
+		other.DefaultImagePullBackOffTimeout == cfg.DefaultImagePullBackOffTimeout &&
 		reflect.DeepEqual(other.DefaultForbiddenEnv, cfg.DefaultForbiddenEnv)
 }
 
@@ -117,6 +122,7 @@ func NewDefaultsFromMap(cfgMap map[string]string) (*Defaults, error) {
 		DefaultCloudEventsSink:            DefaultCloudEventSinkValue,
 		DefaultMaxMatrixCombinationsCount: DefaultMaxMatrixCombinationsCount,
 		DefaultResolverType:               DefaultResolverTypeValue,
+		DefaultImagePullBackOffTimeout:    DefaultImagePullBackOffTimeout,
 	}
 
 	if defaultTimeoutMin, ok := cfgMap[defaultTimeoutMinutesKey]; ok {
@@ -179,6 +185,14 @@ func NewDefaultsFromMap(cfgMap map[string]string) (*Defaults, error) {
 		tc.DefaultResolverType = defaultResolverType
 	}
 
+	if defaultImagePullBackOff, ok := cfgMap[defaultImagePullBackOffTimeout]; ok {
+		timeout, err := time.ParseDuration(defaultImagePullBackOff)
+		if err != nil {
+			return nil, fmt.Errorf("failed parsing tracing config %q", defaultImagePullBackOffTimeout)
+		}
+		tc.DefaultImagePullBackOffTimeout = timeout
+	}
+
 	return &tc, nil
 }
 

diff --git a/pkg/apis/config/default_test.go b/pkg/apis/config/default_test.go
@@ -18,6 +18,7 @@ package config_test
 
 import (
 	"testing"
+	"time"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/tektoncd/pipeline/pkg/apis/config"
@@ -41,6 +42,7 @@ func TestNewDefaultsFromConfigMap(t *testing.T) {
 				DefaultManagedByLabelValue:        "something-else",
 				DefaultMaxMatrixCombinationsCount: 256,
 				DefaultResolverType:               "git",
+				DefaultImagePullBackOffTimeout:    time.Duration(5) * time.Second,
 			},
 			fileName: config.GetDefaultsConfigName(),
 		},
@@ -60,12 +62,16 @@ func TestNewDefaultsFromConfigMap(t *testing.T) {
 					},
 				},
 				DefaultMaxMatrixCombinationsCount: 256,
+				DefaultImagePullBackOffTimeout:    0,
 			},
 			fileName: "config-defaults-with-pod-template",
 		},
 		{
 			expectedError: true,
 			fileName:      "config-defaults-timeout-err",
+		}, {
+			expectedError: true,
+			fileName:      "config-defaults-imagepullbackoff-timeout-err",
 		},
 		// Previously the yaml package did not support UnmarshalStrict, though
 		// it's supported now however it may introduce incompatibility, so we decide
@@ -79,6 +85,7 @@ func TestNewDefaultsFromConfigMap(t *testing.T) {
 				DefaultManagedByLabelValue:        config.DefaultManagedByLabelValue,
 				DefaultPodTemplate:                &pod.Template{},
 				DefaultMaxMatrixCombinationsCount: 256,
+				DefaultImagePullBackOffTimeout:    0,
 			},
 		},
 		{
@@ -90,6 +97,7 @@ func TestNewDefaultsFromConfigMap(t *testing.T) {
 				DefaultManagedByLabelValue:        config.DefaultManagedByLabelValue,
 				DefaultAAPodTemplate:              &pod.AffinityAssistantTemplate{},
 				DefaultMaxMatrixCombinationsCount: 256,
+				DefaultImagePullBackOffTimeout:    0,
 			},
 		},
 		{
@@ -104,6 +112,7 @@ func TestNewDefaultsFromConfigMap(t *testing.T) {
 				DefaultTimeoutMinutes:             60,
 				DefaultServiceAccount:             "default",
 				DefaultManagedByLabelValue:        config.DefaultManagedByLabelValue,
+				DefaultImagePullBackOffTimeout:    0,
 			},
 		},
 		{
@@ -115,6 +124,7 @@ func TestNewDefaultsFromConfigMap(t *testing.T) {
 				DefaultMaxMatrixCombinationsCount: 256,
 				DefaultManagedByLabelValue:        "tekton-pipelines",
 				DefaultForbiddenEnv:               []string{"TEKTON_POWER_MODE", "TEST_ENV", "TEST_TEKTON"},
+				DefaultImagePullBackOffTimeout:    time.Duration(15) * time.Second,
 			},
 		},
 	}
@@ -137,6 +147,7 @@ func TestNewDefaultsFromEmptyConfigMap(t *testing.T) {
 		DefaultManagedByLabelValue:        "tekton-pipelines",
 		DefaultServiceAccount:             "default",
 		DefaultMaxMatrixCombinationsCount: 256,
+		DefaultImagePullBackOffTimeout:    0,
 	}
 	verifyConfigFileWithExpectedConfig(t, DefaultsConfigEmptyName, expectedConfig)
 }
@@ -285,6 +296,24 @@ func TestEquals(t *testing.T) {
 				DefaultForbiddenEnv: []string{"TEST_ENV", "TEKTON_POWER_MODE"},
 			},
 			expected: true,
+		}, {
+			name: "different default ImagePullBackOff timeout",
+			left: &config.Defaults{
+				DefaultImagePullBackOffTimeout: 10,
+			},
+			right: &config.Defaults{
+				DefaultImagePullBackOffTimeout: 20,
+			},
+			expected: false,
+		}, {
+			name: "same default ImagePullBackOff timeout",
+			left: &config.Defaults{
+				DefaultImagePullBackOffTimeout: 20,
+			},
+			right: &config.Defaults{
+				DefaultImagePullBackOffTimeout: 20,
+			},
+			expected: true,
 		},
 	}
 

diff --git a/pkg/apis/config/testdata/config-defaults-forbidden-env.yaml b/pkg/apis/config/testdata/config-defaults-forbidden-env.yaml
@@ -21,3 +21,4 @@ data:
   default-timeout-minutes: "50"
   default-service-account: "tekton"
   default-forbidden-env: "TEST_TEKTON, TEKTON_POWER_MODE,TEST_ENV,TEST_TEKTON"
+  default-imagepullbackoff-timeout: "15s"
diff --git a/pkg/apis/config/testdata/config-defaults-imagepullbackoff-timeout-err.yaml b/pkg/apis/config/testdata/config-defaults-imagepullbackoff-timeout-err.yaml
@@ -0,0 +1,21 @@
+# Copyright 2019 The Tekton Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config-defaults
+  namespace: tekton-pipelines
+data:
+  default-imagepullbackoff-timeout: "not-a-timeout"
diff --git a/pkg/apis/config/testdata/config-defaults.yaml b/pkg/apis/config/testdata/config-defaults.yaml
@@ -22,3 +22,4 @@ data:
   default-service-account: "tekton"
   default-managed-by-label-value: "something-else"
   default-resolver-type: "git"
+  default-imagepullbackoff-timeout: "5s"
diff --git a/pkg/reconciler/taskrun/taskrun.go b/pkg/reconciler/taskrun/taskrun.go
@@ -93,13 +93,15 @@ type Reconciler struct {
 	tracerProvider           trace.TracerProvider
 }
 
+const ImagePullBackOff = "ImagePullBackOff"
+
 var (
 	// Check that our Reconciler implements taskrunreconciler.Interface
 	_ taskrunreconciler.Interface = (*Reconciler)(nil)
 
 	// Pod failure reasons that trigger failure of the TaskRun
 	podFailureReasons = map[string]struct{}{
-		"ImagePullBackOff": {},
+		ImagePullBackOff:   {},
 		"InvalidImageName": {},
 	}
 )
@@ -170,7 +172,7 @@ func (c *Reconciler) ReconcileKind(ctx context.Context, tr *v1.TaskRun) pkgrecon
 	}
 
 	// Check for Pod Failures
-	if failed, reason, message := c.checkPodFailed(tr); failed {
+	if failed, reason, message := c.checkPodFailed(ctx, tr); failed {
 		err := c.failTaskRun(ctx, tr, reason, message)
 		return c.finishReconcileUpdateEmitEvents(ctx, tr, before, err)
 	}
@@ -216,10 +218,30 @@ func (c *Reconciler) ReconcileKind(ctx context.Context, tr *v1.TaskRun) pkgrecon
 	return nil
 }
 
-func (c *Reconciler) checkPodFailed(tr *v1.TaskRun) (bool, v1.TaskRunReason, string) {
+func (c *Reconciler) checkPodFailed(ctx context.Context, tr *v1.TaskRun) (bool, v1.TaskRunReason, string) {
 	for _, step := range tr.Status.Steps {
 		if step.Waiting != nil {
 			if _, found := podFailureReasons[step.Waiting.Reason]; found {
+				if step.Waiting.Reason == ImagePullBackOff {
+					imagePullBackOffTimeOut := config.FromContextOrDefaults(ctx).Defaults.DefaultImagePullBackOffTimeout
+					// only attempt to recover from the imagePullBackOff if specified
+					if imagePullBackOffTimeOut.Seconds() != 0 {
+						p, err := c.KubeClientSet.CoreV1().Pods(tr.Namespace).Get(ctx, tr.Status.PodName, metav1.GetOptions{})
+						if err != nil {
+							message := fmt.Sprintf(`The step %q in TaskRun %q failed to pull the image %q and the pod with error: "%s."`, step.Name, tr.Name, step.ImageID, err)
+							return true, v1.TaskRunReasonImagePullFailed, message
+						}
+						for _, condition := range p.Status.Conditions {
+							// check the pod condition to get the time when the pod was scheduled
+							// keep trying until the pod schedule time has exceeded the specified imagePullBackOff timeout duration
+							if condition.Type == corev1.PodScheduled {
+								if c.Clock.Since(condition.LastTransitionTime.Time) < imagePullBackOffTimeOut {
+									return false, "", ""
+								}
+							}
+						}
+					}
+				}
 				image := step.ImageID
 				message := fmt.Sprintf(`The step %q in TaskRun %q failed to pull the image %q. The pod errored with the message: "%s."`, step.Name, tr.Name, image, step.Waiting.Message)
 				return true, v1.TaskRunReasonImagePullFailed, message
@@ -229,6 +251,26 @@ func (c *Reconciler) checkPodFailed(tr *v1.TaskRun) (bool, v1.TaskRunReason, str
 	for _, sidecar := range tr.Status.Sidecars {
 		if sidecar.Waiting != nil {
 			if _, found := podFailureReasons[sidecar.Waiting.Reason]; found {
+				if sidecar.Waiting.Reason == ImagePullBackOff {
+					imagePullBackOffTimeOut := config.FromContextOrDefaults(ctx).Defaults.DefaultImagePullBackOffTimeout
+					// only attempt to recover from the imagePullBackOff if specified
+					if imagePullBackOffTimeOut.Seconds() != 0 {
+						p, err := c.KubeClientSet.CoreV1().Pods(tr.Namespace).Get(ctx, tr.Status.PodName, metav1.GetOptions{})
+						if err != nil {
+							message := fmt.Sprintf(`The sidecar %q in TaskRun %q failed to pull the image %q and the pod with error: "%s."`, sidecar.Name, tr.Name, sidecar.ImageID, err)
+							return true, v1.TaskRunReasonImagePullFailed, message
+						}
+						for _, condition := range p.Status.Conditions {
+							// check the pod condition to get the time when the pod was scheduled
+							// keep trying until the pod schedule time has exceeded the specified imagePullBackOff timeout duration
+							if condition.Type == corev1.PodScheduled {
+								if c.Clock.Since(condition.LastTransitionTime.Time) < imagePullBackOffTimeOut {
+									return false, "", ""
+								}
+							}
+						}
+					}
+				}
 				image := sidecar.ImageID
 				message := fmt.Sprintf(`The sidecar %q in TaskRun %q failed to pull the image %q. The pod errored with the message: "%s."`, sidecar.Name, tr.Name, image, sidecar.Waiting.Message)
 				return true, v1.TaskRunReasonImagePullFailed, message