Retry tasks on exit code 128 to work around intermittent issues with …

…DNS failures

This should help make mozilla#549 less painful. I suggest we back it out once we get to the bottom of that.

.taskcluster.yml

@@ -227,6 +227,10 @@ tasks:
 
                               image: mozillareleases/taskgraph:decision-v7.4.0@sha256:eb13ec9b67c93c8394856813bbe69bb876888a5015938d8d811ef3609c00fe21
                               maxRunTime: 1800
+                              onExitStatus:
+                                retry:
+                                  # 128 happens when cloning this repository fails
+                                  - 128
 
                               command:
                                   - run-task

taskcluster/kinds/alignments-backtranslated/kind.yml

@@ -57,6 +57,8 @@ tasks:
             env:
                 SRC: "{src_locale}"
                 TRG: "{trg_locale}"
+            # 128 happens when cloning this repository fails
+            retry-exit-status: [128]
 
         # Don't run unless explicitly scheduled
         run-on-tasks-for: []

taskcluster/kinds/alignments-original/kind.yml

@@ -56,6 +56,8 @@ tasks:
             env:
                 SRC: "{src_locale}"
                 TRG: "{trg_locale}"
+            # 128 happens when cloning this repository fails
+            retry-exit-status: [128]
 
         # Don't run unless explicitly scheduled
         run-on-tasks-for: []

taskcluster/kinds/alignments-student/kind.yml

@@ -55,6 +55,8 @@ tasks:
             env:
                 SRC: "{src_locale}"
                 TRG: "{trg_locale}"
+            # 128 happens when cloning this repository fails
+            retry-exit-status: [128]
 
         # Don't run unless explicitly scheduled
         run-on-tasks-for: []

taskcluster/kinds/analyze-corpus/kind.yml

@@ -49,6 +49,8 @@ tasks:
                   path: /builds/worker/artifacts
                   type: directory
             env: {}
+            # 128 happens when cloning this repository fails
+            retry-exit-status: [128]
 
         # Don't run unless explicitly scheduled
         run-on-tasks-for: []

taskcluster/kinds/analyze-mono/kind.yml

@@ -40,6 +40,8 @@ task-defaults:
               path: /builds/worker/artifacts
               type: directory
         env: {}
+        # 128 happens when cloning this repository fails
+        retry-exit-status: [128]
 
     # Don't run unless explicitly scheduled
     run-on-tasks-for: []

taskcluster/kinds/bicleaner-model/kind.yml

@@ -46,6 +46,8 @@ tasks:
                   path: /builds/worker/artifacts
                   type: directory
             max-run-time: 86400
+            # 128 happens when cloning this repository fails
+            retry-exit-status: [128]
 
 
         # Don't run unless explicitly scheduled

taskcluster/kinds/bicleaner/kind.yml

@@ -79,6 +79,8 @@ tasks:
                 CUDNN_DIR: fetches/cuda-toolkit
                 COMPRESSION_CMD: zstdmt
                 ARTIFACT_EXT: zst
+            # 128 happens when cloning this repository fails
+            retry-exit-status: [128]
 
         # Don't run unless explicitly scheduled
         run-on-tasks-for: []

taskcluster/kinds/cefilter/kind.yml

@@ -54,6 +54,8 @@ tasks:
                 ARTIFACT_EXT: zst
                 SRC: "{src_locale}"
                 TRG: "{trg_locale}"
+            # 128 happens when cloning this repository fails
+            retry-exit-status: [128]
 
         # Don't run unless explicitly scheduled
         run-on-tasks-for: []

taskcluster/kinds/clean-corpus/kind.yml

@@ -79,6 +79,8 @@ tasks:
             # there might be intermittent issues with OpusCleaner, specifically FastText model downloading can fail
             retry-exit-status:
                 - 1
+                - 17
+                - 128
 
         # Don't run unless explicitly scheduled
         run-on-tasks-for: []

taskcluster/kinds/clean-mono/kind.yml

@@ -66,6 +66,8 @@ task-defaults:
         env:
             COMPRESSION_CMD: zstdmt
             ARTIFACT_EXT: zst
+        # 128 happens when cloning this repository fails
+        retry-exit-status: [128]
 
     # Don't run unless explicitly scheduled
     run-on-tasks-for: []

taskcluster/kinds/collect-corpus/kind.yml

@@ -64,6 +64,8 @@ tasks:
                   type: directory
             env:
                 COMPRESSION_CMD: zstdmt
+            # 128 happens when cloning this repository fails
+            retry-exit-status: [128]
 
         run:
             using: run-task

taskcluster/kinds/collect-mono-src/kind.yml

@@ -45,6 +45,8 @@ task-defaults:
               type: directory
         env:
             COMPRESSION_CMD: zstdmt
+        # 128 happens when cloning this repository fails
+        retry-exit-status: [128]
 
     # Don't run unless explicitly scheduled
     run-on-tasks-for: []

taskcluster/kinds/collect-mono-trg/kind.yml

@@ -45,6 +45,8 @@ task-defaults:
               type: directory
         env:
             COMPRESSION_CMD: zstdmt
+        # 128 happens when cloning this repository fails
+        retry-exit-status: [128]
 
     # Don't run unless explicitly scheduled
     run-on-tasks-for: []

taskcluster/kinds/dataset/kind.yml

@@ -41,6 +41,8 @@ task-defaults:
             - name: public/build
               path: /builds/worker/artifacts
               type: directory
+        # 128 happens when cloning this repository fails
+        retry-exit-status: [128]
 
     run-on-tasks-for: []
     run:

taskcluster/kinds/evaluate-quantized/kind.yml

@@ -70,6 +70,8 @@ tasks:
 
             # Taskcluster proxy is required to read secrets
             taskcluster-proxy: true
+            # 128 happens when cloning this repository fails
+            retry-exit-status: [128]
 
         # The task needs to be able to read that secret to publish on Weight & Biases
         scopes:

taskcluster/kinds/evaluate-teacher-ensemble/kind.yml

@@ -69,6 +69,8 @@ tasks:
 
             # Taskcluster proxy is required to read secrets
             taskcluster-proxy: true
+            # 128 happens when cloning this repository fails
+            retry-exit-status: [128]
 
         # The task needs to be able to read that secret to publish on Weight & Biases
         scopes:

taskcluster/kinds/evaluate/kind.yml

@@ -67,6 +67,8 @@ task-defaults:
 
         # Taskcluster proxy is required to read secrets
         taskcluster-proxy: true
+        # 128 happens when cloning this repository fails
+        retry-exit-status: [128]
 
     # The task needs to be able to read that secret to publish on Weight & Biases
     scopes:

taskcluster/kinds/export/kind.yml

@@ -52,6 +52,8 @@ tasks:
             env:
                 SRC: "{src_locale}"
                 TRG: "{trg_locale}"
+            # 128 happens when cloning this repository fails
+            retry-exit-status: [128]
 
         # Don't run unless explicitly scheduled
         run-on-tasks-for: []

taskcluster/kinds/extract-best/kind.yml

@@ -68,6 +68,8 @@ tasks:
                   type: directory
             env:
                 COMPRESSION_CMD: zstdmt
+            # 128 happens when cloning this repository fails
+            retry-exit-status: [128]
 
         # Don't run unless explicitly scheduled
         run-on-tasks-for: []

taskcluster/kinds/finetune-student/kind.yml

@@ -62,7 +62,8 @@ tasks:
         worker:
             max-run-time: 2592000
             # train_taskcluster.py exits with 17 if a request to Taskcluster fails
-            retry-exit-status: [17]
+            # 128 happens when cloning this repository fails
+            retry-exit-status: [17, 128]
             env:
                 ARTIFACT_EXT: zst
                 COMPRESSION_CMD: zstdmt

taskcluster/kinds/merge-corpus/kind.yml

@@ -54,6 +54,8 @@ task-defaults:
             TRG: "{trg_locale}"
             COMPRESSION_CMD: zstdmt
             ARTIFACT_EXT: zst
+        # 128 happens when cloning this repository fails
+        retry-exit-status: [128]
 
     # Don't run unless explicitly scheduled
     run-on-tasks-for: []

taskcluster/kinds/merge-devset/kind.yml

@@ -54,6 +54,8 @@ task-defaults:
             TRG: "{trg_locale}"
             COMPRESSION_CMD: zstdmt
             ARTIFACT_EXT: zst
+        # 128 happens when cloning this repository fails
+        retry-exit-status: [128]
 
     # Don't run unless explicitly scheduled
     run-on-tasks-for: []

taskcluster/kinds/merge-mono/kind.yml

@@ -58,6 +58,8 @@ task-defaults:
         env:
             LOCALE: "{locale}"
             COMPRESSION_CMD: zstdmt
+        # 128 happens when cloning this repository fails
+        retry-exit-status: [128]
 
     # Don't run unless explicitly scheduled
     run-on-tasks-for: []

taskcluster/kinds/merge-translated/kind.yml

@@ -52,6 +52,8 @@ task-defaults:
             TRG: "{trg_locale}"
             COMPRESSION_CMD: zstdmt
             ARTIFACT_EXT: zst
+        # 128 happens when cloning this repository fails
+        retry-exit-status: [128]
 
     # Don't run unless explicitly scheduled
     run-on-tasks-for: []

taskcluster/kinds/quantize/kind.yml

@@ -57,6 +57,8 @@ tasks:
             env:
                 SRC: "{src_locale}"
                 TRG: "{trg_locale}"
+            # 128 happens when cloning this repository fails
+            retry-exit-status: [128]
 
         # Don't run unless explicitly scheduled
         run-on-tasks-for: []

taskcluster/kinds/score/kind.yml

@@ -62,6 +62,8 @@ tasks:
                 - name: public/build
                   path: artifacts
                   type: directory
+            # 128 happens when cloning this repository fails
+            retry-exit-status: [128]
 
         # Don't run unless explicitly scheduled
         run-on-tasks-for: []

taskcluster/kinds/shortlist/kind.yml

@@ -58,6 +58,8 @@ tasks:
                 ARTIFACT_EXT: zst
                 SRC: "{src_locale}"
                 TRG: "{trg_locale}"
+            # 128 happens when cloning this repository fails
+            retry-exit-status: [128]
 
         # Don't run unless explicitly scheduled
         run-on-tasks-for: []

taskcluster/kinds/split-corpus/kind.yml

@@ -52,6 +52,8 @@ tasks:
             env:
                 COMPRESSION_CMD: zstdmt
                 ARTIFACT_EXT: zst
+            # 128 happens when cloning this repository fails
+            retry-exit-status: [128]
 
         # Don't run unless explicitly scheduled
         run-on-tasks-for: []

taskcluster/kinds/split-mono-src/kind.yml

@@ -51,6 +51,8 @@ task-defaults:
             LOCALE: "{locale}"
             COMPRESSION_CMD: zstdmt
             ARTIFACT_EXT: zst
+        # 128 happens when cloning this repository fails
+        retry-exit-status: [128]
 
     # Don't run unless explicitly scheduled
     run-on-tasks-for: []

taskcluster/kinds/split-mono-trg/kind.yml

@@ -51,6 +51,8 @@ task-defaults:
             LOCALE: "{locale}"
             COMPRESSION_CMD: zstdmt
             ARTIFACT_EXT: zst
+        # 128 happens when cloning this repository fails
+        retry-exit-status: [128]
 
     # Don't run unless explicitly scheduled
     run-on-tasks-for: []

taskcluster/kinds/tests/kind.yml

@@ -24,6 +24,9 @@ task-defaults:
   run:
     using: run-task
     cwd: '{checkout}'
+  worker:
+    # 128 happens when cloning this repository fails
+    retry-exit-status: [128]
 
 tasks:
   # See issue: https://github.com/mozilla/firefox-translations-training/issues/363

taskcluster/kinds/toolchain/kind.yml

@@ -19,6 +19,8 @@ task-defaults:
         docker-image: {"in-tree": "toolchain-build"}
         max-run-time: 3600
         env: {}
+        # 128 happens when cloning this repository fails
+        retry-exit-status: [128]
     run:
         using: toolchain-script
 

taskcluster/kinds/train-backwards/kind.yml

@@ -79,6 +79,8 @@ tasks:
 
             # Taskcluster proxy is required to read secrets
             taskcluster-proxy: true
+            # 128 happens when cloning this repository fails
+            retry-exit-status: [128]
 
         # The task needs to be able to read that secret to publish on Weight & Biases
         scopes:

taskcluster/kinds/train-student/kind.yml

@@ -60,7 +60,8 @@ tasks:
         worker:
             max-run-time: 2592000
             # train_taskcluster.py exits with 17 if a request to Taskcluster fails
-            retry-exit-status: [17]
+            # 128 happens when cloning this repository fails
+            retry-exit-status: [17, 128]
             env:
                 ARTIFACT_EXT: zst
                 COMPRESSION_CMD: zstdmt

taskcluster/kinds/train-teacher/kind.yml

@@ -85,7 +85,8 @@ tasks:
         worker:
             max-run-time: 2592000
             # train_taskcluster.py exits with 17 if a request to Taskcluster fails
-            retry-exit-status: [17]
+            # 128 happens when cloning this repository fails
+            retry-exit-status: [17, 128]
             env:
                 ARTIFACT_EXT: zst
                 COMPRESSION_CMD: zstdmt

taskcluster/kinds/train-vocab/kind.yml

@@ -54,6 +54,8 @@ tasks:
                   type: directory
             env:
                 COMPRESSION_CMD: zstdmt
+            # 128 happens when cloning this repository fails
+            retry-exit-status: [128]
 
         # Don't run unless explicitly scheduled
         run-on-tasks-for: []

taskcluster/kinds/translate-corpus/kind.yml

@@ -99,6 +99,8 @@ tasks:
             env:
                 CUDA_DIR: fetches/cuda-toolkit
                 CUDNN_DIR: fetches/cuda-toolkit
+            # 128 happens when cloning this repository fails
+            retry-exit-status: [128]
 
         run:
             using: run-task

taskcluster/kinds/translate-mono-src/kind.yml

@@ -52,6 +52,8 @@ task-defaults:
         env:
             CUDA_DIR: fetches/cuda-toolkit
             CUDNN_DIR: fetches/cuda-toolkit
+        # 128 happens when cloning this repository fails
+        retry-exit-status: [128]
 
     # Don't run unless explicitly scheduled
     run-on-tasks-for: []

taskcluster/kinds/translate-mono-trg/kind.yml

@@ -59,6 +59,8 @@ task-defaults:
         env:
             CUDA_DIR: fetches/cuda-toolkit
             CUDNN_DIR: fetches/cuda-toolkit
+        # 128 happens when cloning this repository fails
+        retry-exit-status: [128]
 
     # Don't run unless explicitly scheduled
     run-on-tasks-for: []