From 6b0405db073faae19cc2e692d47e951d9553ec49 Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 9 Oct 2024 21:34:52 +0000 Subject: [PATCH 1/6] p Signed-off-by: kevin --- .../pipeline_generator/pipeline_generator.py | 27 ++++++++++++++++++- .../test_files/test-pipeline.yaml | 23 ++++++++++++++++ .../test_pipeline_generator.py | 24 ++++++++++++++++- 3 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 scripts/tests/pipeline_generator/test_files/test-pipeline.yaml diff --git a/scripts/pipeline_generator/pipeline_generator.py b/scripts/pipeline_generator/pipeline_generator.py index b1053b8..4c912dd 100644 --- a/scripts/pipeline_generator/pipeline_generator.py +++ b/scripts/pipeline_generator/pipeline_generator.py @@ -1,9 +1,11 @@ +import click import os import re +import yaml from typing import List, Optional from pydantic import BaseModel, field_validator - +from .step import TestStep class PipelineGeneratorConfig: def __init__( @@ -53,3 +55,26 @@ def __init__( ): config.validate() self.config = config + +def read_test_steps(file_path: str) -> List[TestStep]: + """Read test steps from test pipeline yaml and parse them into TestStep objects.""" + print(os.getcwd(), os.path.abspath(file_path)) + with open(file_path, "r") as f: + content = yaml.safe_load(f) + return [TestStep(**step) for step in content["steps"]] + +@click.command() +@click.option("--test_path", type=str, required=True, help="Path to the test pipeline yaml file") +@click.option("--run_all", type=str) +@click.option("--list_file_diff", type=str) +def main(test_path: str, external_hardware_test_path: str, run_all: str, list_file_diff: str): + test_steps = read_test_steps(test_path) + + pipeline_generator_config = PipelineGeneratorConfig( + run_all=run_all == "1", + list_file_diff=list_file_diff, + container_registry=VLLM_ECR_URL, + container_registry_repo=VLLM_ECR_REPO, + commit=os.getenv("BUILDKITE_COMMIT"), + pipeline_file_path=PIPELINE_FILE_PATH + ) diff --git a/scripts/tests/pipeline_generator/test_files/test-pipeline.yaml b/scripts/tests/pipeline_generator/test_files/test-pipeline.yaml new file mode 100644 index 0000000..404ea00 --- /dev/null +++ b/scripts/tests/pipeline_generator/test_files/test-pipeline.yaml @@ -0,0 +1,23 @@ +steps: +- label: Test 1 + command: echo "Test 1" + +- label: Test 2 + working_dir: "/tests2/" + no_gpu: true + command: echo "Test 2" + +- label: Test 3 + commands: + - echo "Test 3" + - echo "Test 3.1" + source_file_dependencies: + - "file1" + - "src/file2" + +- label: Test 4 + num_nodes: 2 + num_gpus: 4 + commands: + - echo "Test 4.1" + - echo "Test 4.2" diff --git a/scripts/tests/pipeline_generator/test_pipeline_generator.py b/scripts/tests/pipeline_generator/test_pipeline_generator.py index 145d11e..331abed 100644 --- a/scripts/tests/pipeline_generator/test_pipeline_generator.py +++ b/scripts/tests/pipeline_generator/test_pipeline_generator.py @@ -3,7 +3,8 @@ import os import tempfile -from scripts.pipeline_generator.pipeline_generator import PipelineGeneratorConfig, PipelineGenerator +from scripts.pipeline_generator.pipeline_generator import PipelineGeneratorConfig, PipelineGenerator, read_test_steps +from scripts.pipeline_generator.step import DEFAULT_TEST_WORKING_DIR TEST_COMMIT = "abcdef0123456789abcdef0123456789abcdef01" TEST_FILE_PATH = "tests.yaml" @@ -59,5 +60,26 @@ def test_get_pipeline_generator_fail_nonexistent_test_file(): with pytest.raises(FileNotFoundError, match="Test file"): _ = PipelineGenerator(config) + +def test_read_test_steps(): + current_dir = os.path.dirname(os.path.abspath(__file__)) + test_path = os.path.join(current_dir, "test_files/test-pipeline.yaml") + test_steps = read_test_steps(test_path) + assert len(test_steps) == 4 + assert test_steps[0].commands == ['echo "Test 1"'] + assert test_steps[0].command is None + assert test_steps[0].working_dir == DEFAULT_TEST_WORKING_DIR + + assert test_steps[1].working_dir == "/tests2/" + assert test_steps[1].no_gpu is True + + assert test_steps[2].commands == ['echo "Test 3"', 'echo "Test 3.1"'] + assert test_steps[2].source_file_dependencies == ["file1", "src/file2"] + + assert test_steps[3].commands == ['echo "Test 4.1"', 'echo "Test 4.2"'] + assert test_steps[3].num_nodes == 2 + assert test_steps[3].num_gpus == 4 + + if __name__ == "__main__": sys.exit(pytest.main(["-v", __file__])) From 1c6cef4953c7793a511f2d547f3cda752471865b Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 9 Oct 2024 21:35:51 +0000 Subject: [PATCH 2/6] p Signed-off-by: kevin --- scripts/pipeline_generator/pipeline_generator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/pipeline_generator/pipeline_generator.py b/scripts/pipeline_generator/pipeline_generator.py index 4c912dd..6f318d9 100644 --- a/scripts/pipeline_generator/pipeline_generator.py +++ b/scripts/pipeline_generator/pipeline_generator.py @@ -58,7 +58,6 @@ def __init__( def read_test_steps(file_path: str) -> List[TestStep]: """Read test steps from test pipeline yaml and parse them into TestStep objects.""" - print(os.getcwd(), os.path.abspath(file_path)) with open(file_path, "r") as f: content = yaml.safe_load(f) return [TestStep(**step) for step in content["steps"]] From c0c777e2ce2992c2efbc6428ce11308b469bc15d Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 9 Oct 2024 21:39:33 +0000 Subject: [PATCH 3/6] p Signed-off-by: kevin --- scripts/pipeline_generator/pipeline_generator.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/pipeline_generator/pipeline_generator.py b/scripts/pipeline_generator/pipeline_generator.py index 6f318d9..200d1d5 100644 --- a/scripts/pipeline_generator/pipeline_generator.py +++ b/scripts/pipeline_generator/pipeline_generator.py @@ -14,9 +14,6 @@ def __init__( container_registry_repo: str, commit: str, list_file_diff: List[str], - test_path: str, # List of tests - external_hardware_test_path: str, # List of external hardware tests - pipeline_file_path: str, # Path to the output pipeline file run_all: bool = False, ): self.run_all = run_all @@ -75,5 +72,4 @@ def main(test_path: str, external_hardware_test_path: str, run_all: str, list_fi container_registry=VLLM_ECR_URL, container_registry_repo=VLLM_ECR_REPO, commit=os.getenv("BUILDKITE_COMMIT"), - pipeline_file_path=PIPELINE_FILE_PATH ) From 2ba4de51fb99a8af717093b40353bc5df9c1161d Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 9 Oct 2024 22:26:47 +0000 Subject: [PATCH 4/6] p Signed-off-by: kevin --- scripts/pipeline_generator/pipeline_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/pipeline_generator/pipeline_generator.py b/scripts/pipeline_generator/pipeline_generator.py index 200d1d5..c89dec9 100644 --- a/scripts/pipeline_generator/pipeline_generator.py +++ b/scripts/pipeline_generator/pipeline_generator.py @@ -61,8 +61,8 @@ def read_test_steps(file_path: str) -> List[TestStep]: @click.command() @click.option("--test_path", type=str, required=True, help="Path to the test pipeline yaml file") -@click.option("--run_all", type=str) -@click.option("--list_file_diff", type=str) +@click.option("--run_all", type=str, help="If set to 1, run all tests") +@click.option("--list_file_diff", type=str, help="List of files in the diff between current branch and main") def main(test_path: str, external_hardware_test_path: str, run_all: str, list_file_diff: str): test_steps = read_test_steps(test_path) From cbf17df158aa37e0b38025ac2db78740425f7b99 Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 9 Oct 2024 22:27:47 +0000 Subject: [PATCH 5/6] p Signed-off-by: kevin --- scripts/pipeline_generator/pipeline_generator.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scripts/pipeline_generator/pipeline_generator.py b/scripts/pipeline_generator/pipeline_generator.py index c89dec9..c61eb53 100644 --- a/scripts/pipeline_generator/pipeline_generator.py +++ b/scripts/pipeline_generator/pipeline_generator.py @@ -3,10 +3,10 @@ import re import yaml from typing import List, Optional - from pydantic import BaseModel, field_validator -from .step import TestStep +from .step import TestStep +from .utils import VLLM_ECR_URL, VLLM_ECR_REPO class PipelineGeneratorConfig: def __init__( self, @@ -21,9 +21,6 @@ def __init__( self.container_registry = container_registry self.container_registry_repo = container_registry_repo self.commit = commit - self.test_path = test_path - self.external_hardware_test_path = external_hardware_test_path - self.pipeline_file_path = pipeline_file_path @property def container_image(self): From 76fd37949a27f4fc399f4d9abdb83303a81ea27d Mon Sep 17 00:00:00 2001 From: kevin Date: Wed, 9 Oct 2024 22:30:27 +0000 Subject: [PATCH 6/6] p Signed-off-by: kevin --- .../pipeline_generator/pipeline_generator.py | 8 ---- .../test_pipeline_generator.py | 37 ++++--------------- 2 files changed, 8 insertions(+), 37 deletions(-) diff --git a/scripts/pipeline_generator/pipeline_generator.py b/scripts/pipeline_generator/pipeline_generator.py index c61eb53..0319d7a 100644 --- a/scripts/pipeline_generator/pipeline_generator.py +++ b/scripts/pipeline_generator/pipeline_generator.py @@ -33,14 +33,6 @@ def validate(self): if not re.match(pattern, self.commit): raise ValueError(f"Commit {self.commit} is not a valid Git commit hash") - # Check if test_path exists - if not os.path.isfile(self.test_path): - raise FileNotFoundError(f"Test file {self.test_path} not found") - - # Check if external_hardware_test_path exists - if not os.path.isfile(self.external_hardware_test_path): - raise FileNotFoundError(f"External hardware test file {self.external_hardware_test_path} not found") - class PipelineGenerator: def __init__( diff --git a/scripts/tests/pipeline_generator/test_pipeline_generator.py b/scripts/tests/pipeline_generator/test_pipeline_generator.py index 331abed..5d0696f 100644 --- a/scripts/tests/pipeline_generator/test_pipeline_generator.py +++ b/scripts/tests/pipeline_generator/test_pipeline_generator.py @@ -7,35 +7,23 @@ from scripts.pipeline_generator.step import DEFAULT_TEST_WORKING_DIR TEST_COMMIT = "abcdef0123456789abcdef0123456789abcdef01" -TEST_FILE_PATH = "tests.yaml" -EXTERNAL_HARDWARE_TEST_FILE_PATH = "external_hardware_tests.yaml" -PIPELINE_OUTPUT_FILE_PATH = "pipeline.yaml" TEST_CONTAINER_REGISTRY = "container.registry" TEST_CONTAINER_REGISTRY_REPO = "test" -def _get_pipeline_generator_config(test_dir: str): - with open(os.path.join(test_dir, TEST_FILE_PATH), "w") as f: - f.write("test-content") - with open(os.path.join(test_dir, EXTERNAL_HARDWARE_TEST_FILE_PATH), "w") as f: - f.write("external-hardware-test-content") - +def _get_pipeline_generator_config(): return PipelineGeneratorConfig( container_registry=TEST_CONTAINER_REGISTRY, container_registry_repo=TEST_CONTAINER_REGISTRY_REPO, commit=TEST_COMMIT, list_file_diff=[], - test_path=os.path.join(test_dir, TEST_FILE_PATH), - external_hardware_test_path=os.path.join(test_dir, EXTERNAL_HARDWARE_TEST_FILE_PATH), - pipeline_file_path=os.path.join(test_dir, PIPELINE_OUTPUT_FILE_PATH) ) def test_pipeline_generator_config_get_container_image(): - with tempfile.TemporaryDirectory() as temp_dir: - config = _get_pipeline_generator_config(temp_dir) - config.validate() - assert config.container_image == "container.registry/test:abcdef0123456789abcdef0123456789abcdef01" + config = _get_pipeline_generator_config() + config.validate() + assert config.container_image == "container.registry/test:abcdef0123456789abcdef0123456789abcdef01" @pytest.mark.parametrize( @@ -46,19 +34,10 @@ def test_pipeline_generator_config_get_container_image(): ] ) def test_get_pipeline_generator_config_invalid_commit(commit): - with tempfile.TemporaryDirectory() as temp_dir: - config = _get_pipeline_generator_config(temp_dir) - config.commit = commit - with pytest.raises(ValueError, match="not a valid Git commit hash"): - config.validate() - - -def test_get_pipeline_generator_fail_nonexistent_test_file(): - with tempfile.TemporaryDirectory() as temp_dir: - config = _get_pipeline_generator_config(temp_dir) - config.test_path = "non-existent-file" - with pytest.raises(FileNotFoundError, match="Test file"): - _ = PipelineGenerator(config) + config = _get_pipeline_generator_config() + config.commit = commit + with pytest.raises(ValueError, match="not a valid Git commit hash"): + config.validate() def test_read_test_steps():