Merge branch 'develop' into dockerize_gramine

securefederatedai · Nov 15, 2024 · 9d9a624 · 9d9a624
2 parents e4139ae + 63177f4
commit 9d9a624
Show file tree

Hide file tree

Showing 286 changed files with 278 additions and 172 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -18,9 +18,3 @@ updates:
       interval: "daily"
     ignore:
       - dependency-name: "*"
-  - package-ecosystem: pip
-    directory: /openfl-tutorials/interactive_api
-    schedule:
-      interval: "daily"
-    ignore:
-      - dependency-name: "*"
diff --git a/.github/workflows/task_runner_e2e.yml b/.github/workflows/task_runner_e2e.yml
@@ -1,3 +1,4 @@
+---
 #---------------------------------------------------------------------------
 # Workflow to run Task Runner end to end tests
 # Authors - Noopur, Payal Chaurasiya
@@ -6,16 +7,16 @@ name: Task Runner E2E
 
 on:
   schedule:
-    - cron: '0 0 * * *' # Run every day at midnight
+    - cron: "0 0 * * *" # Run every day at midnight
   workflow_dispatch:
     inputs:
       num_rounds:
-        description: 'Number of rounds to train'
+        description: "Number of rounds to train"
         required: false
         default: "5"
         type: string
       num_collaborators:
-        description: 'Number of collaborators'
+        description: "Number of collaborators"
         required: false
         default: "2"
         type: string
@@ -29,67 +30,132 @@ env:
   NUM_COLLABORATORS: ${{ inputs.num_collaborators || '2' }}
 
 jobs:
-  test_run:
-    name: tr
+  test:
+    name: tr_tls
     runs-on: ubuntu-22.04
     timeout-minutes: 120 # 2 hours
     strategy:
       matrix:
         # There are open issues for some of the models, so excluding them for now:
         # model_name: [ "torch_cnn_mnist", "keras_cnn_mnist", "torch_cnn_histology" ]
-        model_name: [ "torch_cnn_mnist", "keras_cnn_mnist" ]
-        python_version: [ "3.8", "3.9", "3.10" ]
+        model_name: ["torch_cnn_mnist", "keras_cnn_mnist"]
+        python_version: ["3.8", "3.9", "3.10"]
       fail-fast: false # do not immediately fail if one of the combinations fail
 
     env:
       MODEL_NAME: ${{ matrix.model_name }}
       PYTHON_VERSION: ${{ matrix.python_version }}
 
     steps:
-    - name: Checkout OpenFL repository
-      id: checkout_openfl
-      uses: actions/[email protected]
-      with:
-        fetch-depth: 2 # needed for detecting changes
-        submodules: "true"
-        token: ${{ secrets.GITHUB_TOKEN }}
-
-    - name: Set up Python
-      id: setup_python
-      uses: actions/setup-python@v3
-      with:
-        python-version: ${{ env.PYTHON_VERSION }}
-
-    - name: Install dependencies
-      id: install_dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install .
-        pip install -r test-requirements.txt
-
-    - name: Run Task Runner E2E tests
-      id: run_task_runner_tests
-      run: |
-        python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m ${{ env.MODEL_NAME }} --num_rounds $NUM_ROUNDS --num_collaborators $NUM_COLLABORATORS --model_name ${{ env.MODEL_NAME }}
-        echo "Task runner end to end test run completed"
-
-    - name: Print test summary # Print the test summary only if the tests were run
-      id: print_test_summary
-      if: steps.run_task_runner_tests.outcome == 'success' || steps.run_task_runner_tests.outcome == 'failure'
-      run: |
-        export PYTHONPATH="$PYTHONPATH:."
-        python tests/end_to_end/utils/summary_helper.py
-        echo "Test summary printed"
-
-    - name: Tar files # Tar the test results only if the tests were run
-      id: tar_files
-      if: steps.run_task_runner_tests.outcome == 'success' || steps.run_task_runner_tests.outcome == 'failure'
-      run: tar -cvf result.tar results
-
-    - name: Upload Artifacts # Upload the test results only if the tar was created
-      id: upload_artifacts
-      uses: actions/upload-artifact@v4
-      if: steps.tar_files.outcome == 'success'
-      with:
-        name: task_runner_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }}
-        path: result.tar
+      - name: Checkout OpenFL repository
+        id: checkout_openfl
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 2 # needed for detecting changes
+          submodules: "true"
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Python
+        id: setup_python
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install dependencies
+        id: install_dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .
+          pip install -r test-requirements.txt
+
+      - name: Run Task Runner E2E tests with TLS
+        id: run_tests
+        run: |
+          python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m ${{ env.MODEL_NAME }} --num_rounds $NUM_ROUNDS --num_collaborators $NUM_COLLABORATORS --model_name ${{ env.MODEL_NAME }}
+          echo "Task runner end to end test run completed"
+
+      - name: Print test summary # Print the test summary only if the tests were run
+        id: print_test_summary
+        if: steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure'
+        run: |
+          export PYTHONPATH="$PYTHONPATH:."
+          python tests/end_to_end/utils/summary_helper.py
+          echo "Test summary printed"
+
+      - name: Tar files # Tar the test results only if the tests were run
+        id: tar_files
+        if: steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure'
+        run: tar -cvf result.tar results
+
+      - name: Upload Artifacts # Upload the test results only if the tar was created
+        id: upload_artifacts
+        uses: actions/upload-artifact@v4
+        if: steps.tar_files.outcome == 'success'
+        with:
+          name: task_runner_tls_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }}
+          path: result.tar
+
+  test_with_non_tls:
+    name: tr_non_tls
+    runs-on: ubuntu-22.04
+    timeout-minutes: 120 # 2 hours
+    strategy:
+      matrix:
+        # Testing non TLS scenario only for torch_cnn_mnist model and python 3.10
+        # If required, this can be extended to other models and python versions
+        model_name: ["torch_cnn_mnist"]
+        python_version: ["3.10"]
+      fail-fast: false # do not immediately fail if one of the combinations fail
+
+    env:
+      MODEL_NAME: ${{ matrix.model_name }}
+      PYTHON_VERSION: ${{ matrix.python_version }}
+
+    steps:
+      - name: Checkout OpenFL repository
+        id: checkout_openfl
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 2 # needed for detecting changes
+          submodules: "true"
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Python
+        id: setup_python
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install dependencies
+        id: install_dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .
+          pip install -r test-requirements.txt
+
+      - name: Run Task Runner E2E tests without TLS
+        id: run_tests
+        run: |
+          python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m ${{ env.MODEL_NAME }} --num_rounds $NUM_ROUNDS --num_collaborators $NUM_COLLABORATORS --disable_tls
+          echo "Task runner end to end test run completed"
+
+      - name: Print test summary # Print the test summary only if the tests were run
+        id: print_test_summary
+        if: steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure'
+        run: |
+          export PYTHONPATH="$PYTHONPATH:."
+          python tests/end_to_end/utils/summary_helper.py
+          echo "Test summary printed"
+
+      - name: Tar files # Tar the test results only if the tests were run
+        id: tar_files
+        if: steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure'
+        run: tar -cvf result.tar results
+
+      - name: Upload Artifacts # Upload the test results only if the tar was created
+        id: upload_artifacts
+        uses: actions/upload-artifact@v4
+        if: steps.tar_files.outcome == 'success'
+        with:
+          name: task_runner_non_tls_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }}
+          path: result.tar
diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
@@ -47,28 +47,8 @@ jobs:
         coverage run -m pytest -rA
         coverage report
   
-  interactive-kvasir: # from interactive-kvasir.yml
-    needs: [lint, pytest-coverage]
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v3
-      with:
-        python-version: "3.8"
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install .
-    - name: Interactive API - pytorch_kvasir_unet
-      run: |
-        python setup.py build_grpc
-        pip install torch==1.13.1
-        pip install torchvision==0.14.1
-        python -m tests.github.interactive_api_director.experiments.pytorch_kvasir_unet.run
-  
   cli:
-    needs: [lint, pytest-coverage, interactive-kvasir]
+    needs: [lint, pytest-coverage]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -31,28 +31,8 @@ jobs:
         coverage run -m pytest -rA
         coverage report
   
-  interactive-kvasir: # from interactive-kvasir.yml
-    needs: [pytest-coverage]
-    runs-on: windows-latest
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v3
-      with:
-        python-version: "3.8"
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install .
-    - name: Interactive API - pytorch_kvasir_unet
-      run: |
-        python setup.py build_grpc
-        pip install torch==1.13.1
-        pip install torchvision==0.14.1
-        python -m tests.github.interactive_api_director.experiments.pytorch_kvasir_unet.run
-  
   cli: # from taskrunner.yml
-    needs: [pytest-coverage, interactive-kvasir]
+    needs: [pytest-coverage]
     runs-on: windows-latest
     steps:
       - uses: actions/checkout@v3

diff --git a/README.md b/README.md
@@ -17,7 +17,7 @@
   <img alt="Coverity Scan Build Status"
        src="https://scan.coverity.com/projects/29040/badge.svg"/>
 </a>
-[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/intel/openfl/blob/develop/openfl-tutorials/interactive_api/numpy_linear_regression/workspace/SingleNotebook.ipynb)
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/intel/openfl/blob/develop/openfl-tutorials/Federated_Pytorch_MNIST_Tutorial.ipynb)
 
 Open Federated Learning (OpenFL) is a Python 3 framework for Federated Learning. OpenFL is designed to be a _flexible_, _extensible_ and _easily learnable_ tool for data scientists. OpenFL is hosted by The Linux Foundation, aims to be community-driven, and welcomes contributions back to the project. 
 
@@ -37,9 +37,6 @@ For more installation options check out the [online documentation](https://openf
 
 OpenFL enables data scientists to set up a federated learning experiment following one of the workflows:
 
-- [Director-based Workflow](https://openfl.readthedocs.io/en/latest/about/features_index/interactive.html):
-Setup long-lived components to run many experiments in series. Recommended for FL research when many changes to model, dataloader, or hyperparameters are expected
-
 - [Aggregator-based Workflow](https://openfl.readthedocs.io/en/latest/about/features_index/taskrunner.html):
 Define an experiment and distribute it manually. All participants can verify model code and [FL plan](https://openfl.readthedocs.io/en/latest/about/features_index/taskrunner.html#federated-learning-plan-fl-plan-settings) prior to execution. The federation is terminated when the experiment is finished
 

diff --git a/docs/about/features.rst b/docs/about/features.rst
@@ -23,7 +23,7 @@ Task Runner
 
         features_index/taskrunner
 
-Interactive
+Interactive (Deprecated)
     Setup long-lived components to run many experiments in series. Recommended for FL research when many changes to model, dataloader, or hyperparameters are expected.
     Formerly known as the director-based workflow.
     For more info see :doc:`features_index/interactive`
@@ -83,7 +83,6 @@ FedCurv
 
     Use :py:class:`openfl.utilities.fedcurv.torch.FedCurv` to override train function using :code:`.get_penalty()`, :code:`.on_train_begin()`, and :code:`.on_train_end()` methods.
     In addition, you should override default :code:`AggregationFunction` of the train task with :class:`openfl.interface.aggregation_functions.FedCurvWeightedAverage`.
-    See :code:`PyTorch_Histology_FedCurv` tutorial in :code:`../openfl-tutorials/interactive_api` directory for more details.
 
 .. _federated_evaluation:
 
@@ -110,4 +109,4 @@ Quantitatively audit data privacy in statistical and machine learning algorithms
     :hidden:
 
     features_index/privacy_meter
-
+
diff --git a/docs/about/features_index/interactive.rst b/docs/about/features_index/interactive.rst
@@ -3,9 +3,9 @@
 
 .. _running_interactive:
 
-================
-Interactive API
-================
+============================
+Interactive API (Deprecated)
+============================
 
 A director-based workflow uses long-lived components in a federation. These components continue to be available to distribute more experiments in the federation.
 
@@ -670,4 +670,4 @@ Assigner with additional validation round:
 
 .. toctree
 ..    overview.how_can_intel_protect_federated_learning
-..    overview.what_is_intel_federated_learning
+..    overview.what_is_intel_federated_learning
diff --git a/docs/about/features_index/taskrunner.rst b/docs/about/features_index/taskrunner.rst
@@ -112,7 +112,7 @@ Each task subsection contains the following:
 .. _running_the_federation_manual:
 
 
-.. _interactive_api:
+.. _interactive_api (Deprecated):