Skip to content

Commit

Permalink
Merge pull request #1148 from noopurintel/develop
Browse files Browse the repository at this point in the history 
Task Runner tests: Adds non-TLS handling
  • Loading branch information
@teoparvanov
teoparvanov authored Nov 14, 2024
2 parents 4aba7ed + 0591cc2 commit 5c3ab73
Show file tree
Hide file tree
Showing 8 changed files with 243 additions and 87 deletions.
174 changes: 120 additions & 54 deletions .github/workflows/task_runner_e2e.yml
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
---
#---------------------------------------------------------------------------
# Workflow to run Task Runner end to end tests
# Authors - Noopur, Payal Chaurasiya
Expand All @@ -6,16 +7,16 @@ name: Task Runner E2E

on:
schedule:
- cron: '0 0 * * *' # Run every day at midnight
- cron: "0 0 * * *" # Run every day at midnight
workflow_dispatch:
inputs:
num_rounds:
description: 'Number of rounds to train'
description: "Number of rounds to train"
required: false
default: "5"
type: string
num_collaborators:
description: 'Number of collaborators'
description: "Number of collaborators"
required: false
default: "2"
type: string
Expand All @@ -29,67 +30,132 @@ env:
NUM_COLLABORATORS: ${{ inputs.num_collaborators || '2' }}

jobs:
test_run:
name: tr
test:
name: tr_tls
runs-on: ubuntu-22.04
timeout-minutes: 120 # 2 hours
strategy:
matrix:
# There are open issues for some of the models, so excluding them for now:
# model_name: [ "torch_cnn_mnist", "keras_cnn_mnist", "torch_cnn_histology" ]
model_name: [ "torch_cnn_mnist", "keras_cnn_mnist" ]
python_version: [ "3.8", "3.9", "3.10" ]
model_name: ["torch_cnn_mnist", "keras_cnn_mnist"]
python_version: ["3.8", "3.9", "3.10"]
fail-fast: false # do not immediately fail if one of the combinations fail

env:
MODEL_NAME: ${{ matrix.model_name }}
PYTHON_VERSION: ${{ matrix.python_version }}

steps:
- name: Checkout OpenFL repository
id: checkout_openfl
uses: actions/[email protected]
with:
fetch-depth: 2 # needed for detecting changes
submodules: "true"
token: ${{ secrets.GITHUB_TOKEN }}

- name: Set up Python
id: setup_python
uses: actions/setup-python@v3
with:
python-version: ${{ env.PYTHON_VERSION }}

- name: Install dependencies
id: install_dependencies
run: |
python -m pip install --upgrade pip
pip install .
pip install -r test-requirements.txt
- name: Run Task Runner E2E tests
id: run_task_runner_tests
run: |
python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m ${{ env.MODEL_NAME }} --num_rounds $NUM_ROUNDS --num_collaborators $NUM_COLLABORATORS --model_name ${{ env.MODEL_NAME }}
echo "Task runner end to end test run completed"
- name: Print test summary # Print the test summary only if the tests were run
id: print_test_summary
if: steps.run_task_runner_tests.outcome == 'success' || steps.run_task_runner_tests.outcome == 'failure'
run: |
export PYTHONPATH="$PYTHONPATH:."
python tests/end_to_end/utils/summary_helper.py
echo "Test summary printed"
- name: Tar files # Tar the test results only if the tests were run
id: tar_files
if: steps.run_task_runner_tests.outcome == 'success' || steps.run_task_runner_tests.outcome == 'failure'
run: tar -cvf result.tar results

- name: Upload Artifacts # Upload the test results only if the tar was created
id: upload_artifacts
uses: actions/upload-artifact@v4
if: steps.tar_files.outcome == 'success'
with:
name: task_runner_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }}
path: result.tar
- name: Checkout OpenFL repository
id: checkout_openfl
uses: actions/[email protected]
with:
fetch-depth: 2 # needed for detecting changes
submodules: "true"
token: ${{ secrets.GITHUB_TOKEN }}

- name: Set up Python
id: setup_python
uses: actions/setup-python@v3
with:
python-version: ${{ env.PYTHON_VERSION }}

- name: Install dependencies
id: install_dependencies
run: |
python -m pip install --upgrade pip
pip install .
pip install -r test-requirements.txt
- name: Run Task Runner E2E tests with TLS
id: run_tests
run: |
python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m ${{ env.MODEL_NAME }} --num_rounds $NUM_ROUNDS --num_collaborators $NUM_COLLABORATORS --model_name ${{ env.MODEL_NAME }}
echo "Task runner end to end test run completed"
- name: Print test summary # Print the test summary only if the tests were run
id: print_test_summary
if: steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure'
run: |
export PYTHONPATH="$PYTHONPATH:."
python tests/end_to_end/utils/summary_helper.py
echo "Test summary printed"
- name: Tar files # Tar the test results only if the tests were run
id: tar_files
if: steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure'
run: tar -cvf result.tar results

- name: Upload Artifacts # Upload the test results only if the tar was created
id: upload_artifacts
uses: actions/upload-artifact@v4
if: steps.tar_files.outcome == 'success'
with:
name: task_runner_tls_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }}
path: result.tar

test_with_non_tls:
name: tr_non_tls
runs-on: ubuntu-22.04
timeout-minutes: 120 # 2 hours
strategy:
matrix:
# Testing non TLS scenario only for torch_cnn_mnist model and python 3.10
# If required, this can be extended to other models and python versions
model_name: ["torch_cnn_mnist"]
python_version: ["3.10"]
fail-fast: false # do not immediately fail if one of the combinations fail

env:
MODEL_NAME: ${{ matrix.model_name }}
PYTHON_VERSION: ${{ matrix.python_version }}

steps:
- name: Checkout OpenFL repository
id: checkout_openfl
uses: actions/[email protected]
with:
fetch-depth: 2 # needed for detecting changes
submodules: "true"
token: ${{ secrets.GITHUB_TOKEN }}

- name: Set up Python
id: setup_python
uses: actions/setup-python@v3
with:
python-version: ${{ env.PYTHON_VERSION }}

- name: Install dependencies
id: install_dependencies
run: |
python -m pip install --upgrade pip
pip install .
pip install -r test-requirements.txt
- name: Run Task Runner E2E tests without TLS
id: run_tests
run: |
python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m ${{ env.MODEL_NAME }} --num_rounds $NUM_ROUNDS --num_collaborators $NUM_COLLABORATORS --disable_tls
echo "Task runner end to end test run completed"
- name: Print test summary # Print the test summary only if the tests were run
id: print_test_summary
if: steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure'
run: |
export PYTHONPATH="$PYTHONPATH:."
python tests/end_to_end/utils/summary_helper.py
echo "Test summary printed"
- name: Tar files # Tar the test results only if the tests were run
id: tar_files
if: steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure'
run: tar -cvf result.tar results

- name: Upload Artifacts # Upload the test results only if the tar was created
id: upload_artifacts
uses: actions/upload-artifact@v4
if: steps.tar_files.outcome == 'success'
with:
name: task_runner_non_tls_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }}
path: result.tar
19 changes: 14 additions & 5 deletions tests/end_to_end/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,15 +36,24 @@ pip install -r test-requirements.txt
To run a specific test case, use below command:

```sh
python -m pytest tests/end_to_end/test_suites/<test_case_filename> -k <marker> -s
python -m pytest -s tests/end_to_end/test_suites/<test_case_filename> -k <test_case_name>
```

** -s will ensure all the logs are printed on screen. Ignore, if not required.

To modify the number of collaborators, rounds to train and/or model name, use below parameters:
1. --num_collaborators
2. --num_rounds
3. --model_name
Below parameters are available for modification:

1. --num_collaborators <int> - to modify the number of collaborators
2. --num_rounds <int> - to modify the number of rounds to train
3. --model_name <str> - to use a specific model
4. --disable_tls - to disable TLS communication (by default it is enabled)
5. --disable_client_auth - to disable the client authentication (by default it is enabled)

For example, to run Task runner with - torch_cnn_mnist model, 3 collaborators, 5 rounds and non-TLS scenario:

```sh
python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py --num_rounds 5 --num_collaborators 3 --model_name torch_cnn_mnist --disable_tls
```

### Output Structure

Expand Down
69 changes: 54 additions & 15 deletions tests/end_to_end/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
# Define a named tuple to store the objects for model owner, aggregator, and collaborators
federation_fixture = collections.namedtuple(
"federation_fixture",
"model_owner, aggregator, collaborators, model_name, workspace_path, results_dir",
"model_owner, aggregator, collaborators, model_name, disable_client_auth, disable_tls, workspace_path, results_dir",
)


Expand Down Expand Up @@ -50,9 +50,18 @@ def pytest_addoption(parser):
"--model_name",
action="store",
type=str,
default=constants.DEFAULT_MODEL_NAME,
help="Model name",
)
parser.addoption(
"--disable_client_auth",
action="store_true",
help="Disable client authentication",
)
parser.addoption(
"--disable_tls",
action="store_true",
help="Disable TLS for communication",
)


@pytest.fixture(scope="session", autouse=True)
Expand Down Expand Up @@ -199,7 +208,7 @@ def pytest_sessionfinish(session, exitstatus):
log.debug(f"Cleared .pytest_cache directory at {cache_dir}")


@pytest.fixture(scope="module")
@pytest.fixture(scope="function")
def fx_federation(request, pytestconfig):
"""
Fixture for federation. This fixture is used to create the model owner, aggregator, and collaborators.
Expand All @@ -211,18 +220,29 @@ def fx_federation(request, pytestconfig):
Returns:
federation_fixture: Named tuple containing the objects for model owner, aggregator, and collaborators
Note: As this is a module level fixture, thus no import is required at test level.
Note: As this is a function level fixture, thus no import is required at test level.
"""
log.info("Fixture for federation setup using Task Runner API on single machine.")
collaborators = []
agg_domain_name = "localhost"

# Parse the command line arguments
args = parse_arguments()
model_name = args.model_name
# Use the model name from the test case name if not provided as a command line argument
model_name = args.model_name if args.model_name else request.node.name.split("test_")[1]
results_dir = args.results_dir or pytestconfig.getini("results_dir")
num_collaborators = args.num_collaborators
num_rounds = args.num_rounds
disable_client_auth = args.disable_client_auth
disable_tls = args.disable_tls

log.info(
f"Running federation setup using Task Runner API on single machine with below configurations:\n"
f"\tNumber of collaborators: {num_collaborators}\n"
f"\tNumber of rounds: {num_rounds}\n"
f"\tModel name: {model_name}\n"
f"\tClient authentication: {not disable_client_auth}\n"
f"\tTLS: {not disable_tls}"
)

# Validate the model name and create the workspace name
if not model_name.upper() in constants.ModelName._member_names_:
Expand All @@ -238,26 +258,42 @@ def fx_federation(request, pytestconfig):
log.error(f"Failed to create the workspace: {e}")
raise e

# Modify and initialize the plan
# Modify the plan
try:
model_owner.modify_plan(new_rounds=num_rounds, num_collaborators=num_collaborators)
model_owner.modify_plan(
new_rounds=num_rounds,
num_collaborators=num_collaborators,
disable_client_auth=disable_client_auth,
disable_tls=disable_tls,
)
except Exception as e:
log.error(f"Failed to modify the plan: {e}")
raise e

# For TLS enabled (default) scenario: when the workspace is certified, the collaborators are registered as well
# For TLS disabled scenario: collaborators need to be registered explicitly
if args.disable_tls:
log.info("Disabling TLS for communication")
try:
model_owner.register_collaborators(num_collaborators)
except Exception as e:
log.error(f"Failed to register the collaborators: {e}")
raise e
else:
log.info("Enabling TLS for communication")
try:
model_owner.certify_workspace()
except Exception as e:
log.error(f"Failed to certify the workspace: {e}")
raise e

# Initialize the plan
try:
model_owner.initialize_plan(agg_domain_name=agg_domain_name)
except Exception as e:
log.error(f"Failed to initialize the plan: {e}")
raise e

# Modify and initialize the plan
try:
model_owner.certify_workspace()
except Exception as e:
log.error(f"Failed to certify the workspace: {e}")
raise e

# Create the objects for aggregator and collaborators
aggregator = participants.Aggregator(
agg_domain_name=agg_domain_name, workspace_path=workspace_path
Expand All @@ -269,6 +305,7 @@ def fx_federation(request, pytestconfig):
data_directory_path=i + 1,
workspace_path=workspace_path,
)
collaborator.create_collaborator()
collaborators.append(collaborator)

# Return the federation fixture
Expand All @@ -277,6 +314,8 @@ def fx_federation(request, pytestconfig):
aggregator=aggregator,
collaborators=collaborators,
model_name=model_name,
disable_client_auth=disable_client_auth,
disable_tls=disable_tls,
workspace_path=workspace_path,
results_dir=results_dir,
)
Loading

0 comments on commit 5c3ab73

Please sign in to comment.