Merge branch 'main' into kdestin/update-readme

Azure · Oct 23, 2023 · c685a64 · c685a64
2 parents 15fd2ef + e5978d3
commit c685a64
Show file tree

Hide file tree

Showing 246 changed files with 30,110 additions and 8,017 deletions.
diff --git a/...re_openai-openai_completions_finetune.yml → ...ai-oai-v1-openai_completions_finetune.yml b/...re_openai-openai_completions_finetune.yml → ...ai-oai-v1-openai_completions_finetune.yml
@@ -3,7 +3,7 @@
 # Any manual changes to this file may cause incorrect behavior.
 # Any manual changes will be overwritten if the code is regenerated.
 
-name: cli-foundation-models-azure_openai-openai_completions_finetune
+name: cli-foundation-models-azure_openai-oai-v1-openai_completions_finetune
 # This file is created by readme.py.
 # Please do not edit directly.
 on:
@@ -14,8 +14,8 @@ on:
     branches:
       - main
     paths:
-      - cli/foundation-models/azure_openai/openai_completions_finetune/**
-      - .github/workflows/cli-foundation-models-azure_openai-openai_completions_finetune.yml
+      - cli/foundation-models/azure_openai/oai-v1/openai_completions_finetune/**
+      - .github/workflows/cli-foundation-models-azure_openai-oai-v1-openai_completions_finetune.yml
       - infra/bootstrapping/**
       - cli/run-pipeline-jobs.sh
       - cli/setup.sh
@@ -52,5 +52,5 @@ jobs:
           bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json";
           [ -f "../../.azureml/config" ] && cat "../../.azureml/config";
           bash -x ../dataset-create.sh
-          bash -x ../../../run-job.sh openai_completions_finetune_spec.yaml
-      working-directory: cli/foundation-models/azure_openai/openai_completions_finetune
+          bash -x ../../../../run-job.sh openai_completions_finetune_spec.yaml
+      working-directory: cli/foundation-models/azure_openai/oai-v1/openai_completions_finetune
diff --git a/.github/workflows/cli-foundation-models-azure_openai-oai-v2-openai_completions_finetune.yml b/.github/workflows/cli-foundation-models-azure_openai-oai-v2-openai_completions_finetune.yml
@@ -0,0 +1,56 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: cli-foundation-models-azure_openai-oai-v2-openai_completions_finetune
+# This file is created by readme.py.
+# Please do not edit directly.
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "23 2/12 * * *"
+  pull_request:
+    branches:
+      - main
+    paths:
+      - cli/foundation-models/azure_openai/oai-v1/openai_completions_finetune_pipeline/**
+      - .github/workflows/cli-foundation-models-azure_openai-oai-v2-openai_completions_finetune.yml
+      - infra/bootstrapping/**
+      - cli/run-pipeline-jobs.sh
+      - cli/setup.sh
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: check out repo
+        uses: actions/checkout@v2
+      - name: azure login
+        uses: azure/login@v1
+        with:
+          creds: ${{secrets.AZUREML_CREDENTIALS}}
+      - name: bootstrap resources
+        run: |
+          echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}';
+          bash bootstrap_oai_v2.sh
+        working-directory: infra/bootstrapping
+        continue-on-error: false
+      - name: setup-cli
+        run: |
+          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/bootstrapping/init_environment_oai_v2.sh";
+          bash setup_oai_v2.sh
+        working-directory: cli
+        continue-on-error: true
+      - name: run job
+        run: |
+          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/bootstrapping/init_environment_oai_v2.sh";
+          bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json";
+          [ -f "../../.azureml/config" ] && cat "../../.azureml/config";
+          bash -x ../dataset-create.sh
+          bash -x ../../../../run-job.sh openai_completions_finetune_pipeline_spec.yaml
+        working-directory: cli/foundation-models/azure_openai/oai-v2/openai_completions_finetune_pipeline
diff --git a/...flows/cli-jobs-pipelines-with-components-pipeline_job_with_flow_as_component-pipeline.yml b/...flows/cli-jobs-pipelines-with-components-pipeline_job_with_flow_as_component-pipeline.yml
@@ -0,0 +1,51 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: cli-jobs-pipelines-with-components-pipeline_job_with_flow_as_component-pipeline
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "4 6/12 * * *"
+  pull_request:
+    branches:
+      - main
+    paths:
+      - cli/jobs/pipelines-with-components/pipeline_job_with_flow_as_component/**
+      - infra/bootstrapping/**
+      - .github/workflows/cli-jobs-pipelines-with-components-pipeline_job_with_flow_as_component-pipeline.yml
+      - cli/run-pipeline-jobs.sh
+      - cli/setup.sh
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: check out repo
+      uses: actions/checkout@v2
+    - name: azure login
+      uses: azure/login@v1
+      with:
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
+    - name: bootstrap resources
+      run: |
+          echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}';
+          bash bootstrap.sh
+      working-directory: infra/bootstrapping
+      continue-on-error: false
+    - name: setup-cli
+      run: |
+          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
+          bash setup.sh
+      working-directory: cli
+      continue-on-error: true
+    - name: run job
+      run: |
+          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
+          bash -x ../../../run-job.sh pipeline.yml
+      working-directory: cli/jobs/pipelines-with-components/pipeline_job_with_flow_as_component
diff --git a/.github/workflows/cli-jobs-spark-attached-spark-pipeline-default-identity.yml b/.github/workflows/cli-jobs-spark-attached-spark-pipeline-default-identity.yml
@@ -52,7 +52,7 @@ jobs:
       working-directory: cli
       continue-on-error: true
       run: |
-          bash -x jobs/spark/setup-attached-resources.sh resources/compute/attached-spark.yml jobs/spark/attached-spark-pipeline-default-identity.yml
+          bash -x jobs/spark/setup-attached-resources.sh resources/compute/attached-spark-system-identity.yml jobs/spark/attached-spark-pipeline-default-identity.yml
     - name: run job
       run: |
           source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";

diff --git a/.github/workflows/cli-jobs-spark-attached-spark-standalone-default-identity.yml b/.github/workflows/cli-jobs-spark-attached-spark-standalone-default-identity.yml
@@ -52,7 +52,7 @@ jobs:
       working-directory: cli
       continue-on-error: true
       run: |
-          bash -x jobs/spark/setup-attached-resources.sh resources/compute/attached-spark.yml jobs/spark/attached-spark-standalone-default-identity.yml
+          bash -x jobs/spark/setup-attached-resources.sh resources/compute/attached-spark-system-identity.yml jobs/spark/attached-spark-standalone-default-identity.yml
     - name: run job
       run: |
           source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";

diff --git a/.github/workflows/code-quality-python.yml b/.github/workflows/code-quality-python.yml
@@ -0,0 +1,17 @@
+name: Python Quality Checks
+
+on:
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - run: pip install -r dev-requirements.txt
+      - name: Check formatting for Python Scripts and Jupyter Notebooks
+        run: black --check .
diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml
@@ -1,10 +1,13 @@
 name: Code Quality Checks
 
 on:
-  workflow_dispatch:
+  push:
+    branches:
+      - main
   pull_request:
     branches:
       - main
+  workflow_dispatch:
 
 jobs:
   pull_request:
@@ -19,3 +22,5 @@ jobs:
         run: |
           git fetch origin ${{ github.event.pull_request.base.ref }} --quiet # Need to manually fetch base branch in CI
           python ./.github/scripts/commit-filesize-diff-summary.py --limit 2MB origin/${{ github.event.pull_request.base.ref }}..HEAD
+  python:
+    uses: ./.github/workflows/code-quality-python.yml
diff --git a/...re_openai-openai_completions_finetune.yml → ...ai-oai-v1-openai_completions_finetune.yml b/...re_openai-openai_completions_finetune.yml → ...ai-oai-v1-openai_completions_finetune.yml
@@ -1,75 +1,75 @@
-# This code is autogenerated.
-# Code is generated by running custom script: python3 readme.py
-# Any manual changes to this file may cause incorrect behavior.
-# Any manual changes will be overwritten if the code is regenerated.
-
-name: sdk-foundation-models-azure_openai-openai_completions_finetune
-# This file is created by sdk/python/readme.py.
-# Please do not edit directly.
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: "23 2/12 * * *"
-  pull_request:
-    branches:
-      - main
-    paths:
-      - sdk/python/foundation-models/azure_openai/**
-      - .github/workflows/sdk-foundation-models-azure_openai-openai_completions_finetune.yml
-      - sdk/python/dev-requirements.txt
-      - infra/bootstrapping/**
-      - sdk/python/setup.sh
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-    - name: check out repo
-      uses: actions/checkout@v2
-    - name: setup python
-      uses: actions/setup-python@v2
-      with:
-        python-version: "3.8"
-    - name: pip install notebook reqs
-      run: pip install -r sdk/python/dev-requirements.txt
-    - name: azure login
-      uses: azure/login@v1
-      with:
-        creds: ${{secrets.AZUREML_CREDENTIALS}}
-    - name: bootstrap resources
-      run: |
-          echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}';
-          bash bootstrap.sh
-      working-directory: infra/bootstrapping
-      continue-on-error: false
-    - name: setup SDK
-      run: |
-          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
-          source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
-          bash setup.sh
-      working-directory: sdk/python
-      continue-on-error: true
-    - name: setup-cli
-      run: |
-          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
-          source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
-          bash setup.sh
-      working-directory: cli
-      continue-on-error: true
-    - name: run foundation-models/azure_openai/openai_completions_finetune.ipynb
-      run: |
-          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
-          source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
-          bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json";
-          bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "openai_completions_finetune.ipynb";
-          [ -f "../../.azureml/config" ] && cat "../../.azureml/config";
-          papermill -k python openai_completions_finetune.ipynb openai_completions_finetune.output.ipynb
-      working-directory: sdk/python/foundation-models/azure_openai
-    - name: upload notebook's working folder as an artifact
-      if: ${{ always() }}
-      uses: actions/upload-artifact@v2
-      with:
-        name: openai_completions_finetune
-        path: sdk/python/foundation-models/azure_openai
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: sdk-foundation-models-azure_openai-oai-v1-openai_completions_finetune
+# This file is created by sdk/python/readme.py.
+# Please do not edit directly.
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "23 2/12 * * *"
+  pull_request:
+    branches:
+      - main
+    paths:
+      - sdk/python/foundation-models/azure_openai/oai-v1/**
+      - .github/workflows/sdk-foundation-models-azure_openai-oai-v1-openai_completions_finetune.yml
+      - sdk/python/dev-requirements.txt
+      - infra/bootstrapping/**
+      - sdk/python/setup.sh
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: check out repo
+      uses: actions/checkout@v2
+    - name: setup python
+      uses: actions/setup-python@v2
+      with:
+        python-version: "3.8"
+    - name: pip install notebook reqs
+      run: pip install -r sdk/python/dev-requirements.txt
+    - name: azure login
+      uses: azure/login@v1
+      with:
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
+    - name: bootstrap resources
+      run: |
+          echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}';
+          bash bootstrap.sh
+      working-directory: infra/bootstrapping
+      continue-on-error: false
+    - name: setup SDK
+      run: |
+          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
+          bash setup.sh
+      working-directory: sdk/python
+      continue-on-error: true
+    - name: setup-cli
+      run: |
+          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
+          bash setup.sh
+      working-directory: cli
+      continue-on-error: true
+    - name: run foundation-models/azure_openai/oai-v1/openai_completions_finetune.ipynb
+      run: |
+          source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/bootstrapping/init_environment.sh";
+          bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json";
+          bash "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh" replace_template_values "openai_completions_finetune.ipynb";
+          [ -f "../../.azureml/config" ] && cat "../../.azureml/config";
+          papermill -k python openai_completions_finetune.ipynb openai_completions_finetune.output.ipynb
+      working-directory: sdk/python/foundation-models/azure_openai/oai-v1
+    - name: upload notebook's working folder as an artifact
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v2
+      with:
+        name: openai_completions_finetune
+        path: sdk/python/foundation-models/azure_openai/oai-v1