Merge branch 'ko3n1g/ci/improve-release-tests' into 'main'

ci: Do not print logs for release tests See merge request ADLR/megatron-lm!2654
NVIDIA · Feb 9, 2025 · 2481987 · 2481987
2 parents cd4a391 + 4d00edb
commit 2481987
Show file tree

Hide file tree

Showing 8 changed files with 23 additions and 13 deletions.
diff --git a/tests/functional_tests/shell_test_utils/_run_training.sh b/tests/functional_tests/shell_test_utils/_run_training.sh
@@ -94,15 +94,15 @@ MASTER_PORT=${MASTER_PORT:-6000}
 NUM_NODES=${NUM_NODES:-${SLURM_NNODES}}
 GPUS_PER_NODE=${GPUS_PER_NODE:-8}
 NODE_RANK=${SLURM_NODEID:-${SLURM_NODEID}}
-LAST_RANK=$((NUM_NODES * 8 - 1))
+LAST_RANK=7
 DISTRIBUTED_ARGS=(
     --nproc_per_node $GPUS_PER_NODE
     --nnodes $NUM_NODES
     --master_addr $MASTER_ADDR
     --master_port $MASTER_PORT
     --node_rank $SLURM_NODEID
     --log-dir $OUTPUT_PATH
-    --tee "0:3,$LAST_RANK:3"
+    --tee "0:3,7:3"
     --redirects "3"
 )
 

diff --git a/tests/functional_tests/shell_test_utils/run_ci_test.sh b/tests/functional_tests/shell_test_utils/run_ci_test.sh
@@ -66,6 +66,11 @@ for i in $(seq 1 $N_REPEAT); do
     export CHECKPOINT_SAVE_PATH=$_CHECKPOINT_SAVE_PATH
     export CHECKPOINT_LOAD_PATH=/tmp/checkpoints/
 
+    if [[ "$TEST_TYPE" = "release" ]]; then
+        export CHECKPOINT_LOAD_PATH=$_CHECKPOINT_LOAD_PATH
+        export CHECKPOINT_SAVE_PATH=$_CHECKPOINT_SAVE_PATH
+    fi
+
     bash $ROOT_DIR/tests/functional_tests/shell_test_utils/_run_training.sh
 
     if [[ "$TEST_TYPE" = "frozen-resume" && -z "$(ls -A "$_CHECKPOINT_LOAD_PATH" 2>/dev/null)" ]]; then
@@ -115,6 +120,10 @@ for i in $(seq 1 $N_REPEAT); do
         --output-path ${OUTPUT_PATH}/$(basename $GOLDEN_VALUES_PATH) \
         "${EXTRACT_ARGS[@]}"
 
+    if [[ "$TEST_TYPE" == "release" ]]; then
+        SKIP_PYTEST=0
+    fi
+
     # Maybe run tests
     if [[ ${SKIP_PYTEST:-0} != 1 ]]; then
         export NVTE_ALLOW_NONDETERMINISTIC_ALGO

diff --git a/tests/functional_tests/test_cases/bert/bert_release/model_config.yaml b/tests/functional_tests/test_cases/bert/bert_release/model_config.yaml
@@ -34,8 +34,8 @@ MODEL_ARGS:
   --log-interval: 100
   --save-interval: 2000
   --eval-interval: 1000
-  --save: ${CHECKPOINT_PATH}
-  --load: ${CHECKPOINT_PATH}
+  --save: ${CHECKPOINT_SAVE_PATH}
+  --load: ${CHECKPOINT_LOAD_PATH}
   --eval-iters: 10
   --tensorboard-dir: ${TENSORBOARD_PATH}
   --log-timers-to-tensorboard: true

diff --git a/tests/functional_tests/test_cases/gpt/gpt3_15b_8t_release/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt3_15b_8t_release/model_config.yaml
@@ -67,8 +67,8 @@ MODEL_ARGS:
   --eval-iters: 32
   --eval-interval: 2000
   # Add checkpointing args
-  --load: ${OUTPUT_PATH}/checkpoints
-  --save: ${OUTPUT_PATH}/checkpoints
+  --save: ${CHECKPOINT_SAVE_PATH}
+  --load: ${CHECKPOINT_LOAD_PATH}
   --save-interval: 5000
   # Add initialization args
   --init-method-std: 0.0134

diff --git a/...tional_tests/test_cases/mixtral/mixtral_8x7b_alltoall_tp2pp4ep4_release/model_config.yaml b/...tional_tests/test_cases/mixtral/mixtral_8x7b_alltoall_tp2pp4ep4_release/model_config.yaml
@@ -76,8 +76,8 @@ MODEL_ARGS:
   --eval-iters: 32
   --eval-interval: 200
   # Add checkpointing args
-  --load: ${OUTPUT_PATH}/checkpoints
-  --save: ${OUTPUT_PATH}/checkpoints
+  --save: ${CHECKPOINT_SAVE_PATH}
+  --load: ${CHECKPOINT_LOAD_PATH}
   --save-interval: 5000
   # Add initialization args
   --init-method-std: 0.010

diff --git a/...nal_tests/test_cases/mixtral/mixtral_8x7b_alltoall_tp2pp4ep4_release_sm/model_config.yaml b/...nal_tests/test_cases/mixtral/mixtral_8x7b_alltoall_tp2pp4ep4_release_sm/model_config.yaml
@@ -76,8 +76,8 @@ MODEL_ARGS:
   --eval-iters: 32
   --eval-interval: 200
   # Add checkpointing args
-  --load: ${OUTPUT_PATH}/checkpoints
-  --save: ${OUTPUT_PATH}/checkpoints
+  --save: ${CHECKPOINT_LOAD_PATH}
+  --load: ${CHECKPOINT_SAVE_PATH}
   --save-interval: 500
   # Add initialization args
   --init-method-std: 0.010

diff --git a/tests/functional_tests/test_cases/t5/t5_release/model_config.yaml b/tests/functional_tests/test_cases/t5/t5_release/model_config.yaml
@@ -45,8 +45,8 @@ MODEL_ARGS:
   --log-interval: 100
   --save-interval: 2000
   --eval-interval: 1000
-  --save: ${CHECKPOINT_PATH}
-  --load: ${CHECKPOINT_PATH}
+  --save: ${CHECKPOINT_SAVE_PATH}
+  --load: ${CHECKPOINT_LOAD_PATH}
   --eval-iters: 10
   --tensorboard-dir: ${TENSORBOARD_PATH}
   --log-timers-to-tensorboard: true

diff --git a/tests/test_utils/python_scripts/launch_jet_workload.py b/tests/test_utils/python_scripts/launch_jet_workload.py
@@ -306,7 +306,8 @@ def main(
             n_attempts += 1
             continue
 
-        print(f"Logs:\n{concat_logs}")
+        if test_type != "release":
+            print(f"Logs:\n{concat_logs}")
 
         success = pipeline.get_status() == PipelineStatus.SUCCESS
         logger.info("Pipeline terminated with status %s", pipeline.get_status().name)