updated skip lists to skip spec decode

neuralmagic · May 13, 2024 · 1c359ae · 1c359ae · github-actions · May 13, 2024
1 parent 9a9c899
commit 1c359ae
Show file tree

Hide file tree

Showing 5 changed files with 5 additions and 1 deletion.
diff --git a/.github/scripts/run-tests b/.github/scripts/run-tests
@@ -83,7 +83,7 @@ fi
 echo "..."
 for EXCLUDE in "${TESTS_TO_EXCLUDE[@]}"; do
     for JJ in "${!TESTS_FOUND[@]}"; do
-        if [[ ${TESTS_FOUND[$JJ]} = {EXCLUDE} ]]; then
+        if [[ ${TESTS_FOUND[$JJ]} = ${EXCLUDE} ]]; then
             echo "excluding: ${EXCLUDE}"
             unset 'TESTS_FOUND[$JJ]'
         fi

diff --git a/neuralmagic/tests/skip-for-nightly.txt b/neuralmagic/tests/skip-for-nightly.txt
@@ -8,6 +8,7 @@ tests/spec_decode/test_utils.py
 tests/spec_decode/test_multi_step_worker.py
 tests/spec_decode/test_spec_decode_worker.py
 tests/spec_decode/test_batch_expansion.py
+tests/spec_decode/test_metrics.py
 tests/spec_decode/test_ngram_worker.py
 tests/spec_decode/e2e/test_logprobs.py
 tests/spec_decode/e2e/test_ngram_correctness.py

diff --git a/neuralmagic/tests/skip-for-release.txt b/neuralmagic/tests/skip-for-release.txt
@@ -8,6 +8,7 @@ tests/spec_decode/test_utils.py
 tests/spec_decode/test_multi_step_worker.py
 tests/spec_decode/test_spec_decode_worker.py
 tests/spec_decode/test_batch_expansion.py
+tests/spec_decode/test_metrics.py
 tests/spec_decode/test_ngram_worker.py
 tests/spec_decode/e2e/test_logprobs.py
 tests/spec_decode/e2e/test_ngram_correctness.py

diff --git a/neuralmagic/tests/skip-for-remote-push.txt b/neuralmagic/tests/skip-for-remote-push.txt
@@ -20,6 +20,7 @@ tests/spec_decode/test_utils.py
 tests/spec_decode/test_multi_step_worker.py
 tests/spec_decode/test_spec_decode_worker.py
 tests/spec_decode/test_batch_expansion.py
+tests/spec_decode/test_metrics.py
 tests/spec_decode/test_ngram_worker.py
 tests/spec_decode/e2e/test_logprobs.py
 tests/spec_decode/e2e/test_ngram_correctness.py

diff --git a/neuralmagic/tests/skip-for-weekly.txt b/neuralmagic/tests/skip-for-weekly.txt
@@ -8,6 +8,7 @@ tests/spec_decode/test_utils.py
 tests/spec_decode/test_multi_step_worker.py
 tests/spec_decode/test_spec_decode_worker.py
 tests/spec_decode/test_batch_expansion.py
+tests/spec_decode/test_metrics.py
 tests/spec_decode/test_ngram_worker.py
 tests/spec_decode/e2e/test_logprobs.py
 tests/spec_decode/e2e/test_ngram_correctness.py
Benchmark suite	Current: `1c359ae`	Previous: `df1f1a0`	Ratio
`{"name": "request_throughput", "description": "VLLM Engine throughput - synthetic\nmodel - NousResearch/Llama-2-7b-chat-hf\nmax_model_len - 4096\nbenchmark_throughput {\n \"use-all-available-gpus_\": \"\",\n \"input-len\": 256,\n \"output-len\": 128,\n \"num-prompts\": 1000\n}", "gpu_description": "NVIDIA A10G x 1", "vllm_version": "0.2.0", "python_version": "3.10.12 (main, May 10 2024, 13:42:25) [GCC 9.4.0]", "torch_version": "2.3.0+cu121"}`	`3.834722011280784` prompts/s
`{"name": "token_throughput", "description": "VLLM Engine throughput - synthetic\nmodel - NousResearch/Llama-2-7b-chat-hf\nmax_model_len - 4096\nbenchmark_throughput {\n \"use-all-available-gpus_\": \"\",\n \"input-len\": 256,\n \"output-len\": 128,\n \"num-prompts\": 1000\n}", "gpu_description": "NVIDIA A10G x 1", "vllm_version": "0.2.0", "python_version": "3.10.12 (main, May 10 2024, 13:42:25) [GCC 9.4.0]", "torch_version": "2.3.0+cu121"}`	`1472.5332523318211` tokens/s