From 706e913bc60ed39fad8f16bd14340c9f33606c61 Mon Sep 17 00:00:00 2001 From: Daniel Ciborowski Date: Wed, 15 Jan 2020 16:45:32 -0500 Subject: [PATCH 01/20] Update azure-pipelines.yml --- .ci/azure-pipelines.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 434c9ab..92f8721 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -32,6 +32,7 @@ jobs: az login -t $(sptenent) --service-principal -u $(spidentity) --password $(spsecret) az account set --subscription $(subscriptionid) displayName: 'Build Configuration' + failOnStderr: true - bash: | source activate deployment_aml @@ -46,18 +47,23 @@ jobs: -p subscription_id $(subscriptionid) \ -p image_name 'modelimg' displayName : '00_AMLSetup.ipynb' + failOnStderr: true + - template: steps/papermill.yml parameters: notebook: 01_DevelopModel.ipynb location: "{{cookiecutter.project_name}}/Keras_Tensorflow" + - template: steps/papermill.yml parameters: notebook: 02_DevelopModelDriver.ipynb location: "{{cookiecutter.project_name}}/Keras_Tensorflow" + - template: steps/papermill.yml parameters: notebook: 03_BuildImage.ipynb location: "{{cookiecutter.project_name}}/Keras_Tensorflow" + - bash: | source activate deployment_aml export PYTHONPATH=$(pwd)/{{cookiecutter.project_name}}/Keras_Tensorflow:${PYTHONPATH} @@ -71,14 +77,18 @@ jobs: -p aks_location $(azureregion) sleep 1m displayName : '04_DeployOnAKS.ipynb' + failOnStderr: true + - template: steps/papermill.yml parameters: notebook: 05_TestWebApp.ipynb location: "{{cookiecutter.project_name}}/Keras_Tensorflow/aks" + - template: steps/papermill.yml parameters: notebook: 06_SpeedTestWebApp.ipynb location: "{{cookiecutter.project_name}}/Keras_Tensorflow/aks" + # - bash: | # source activate deployment_aml # export PYTHONPATH=$(pwd)/{{cookiecutter.project_name}}/Keras_Tensorflow:${PYTHONPATH} @@ -96,6 +106,7 @@ jobs: parameters: notebook: 07_TearDown.ipynb location: "{{cookiecutter.project_name}}/Keras_Tensorflow/aks" + # - template: steps/papermill.yml # parameters: # notebook: 05_TearDown.ipynb From ccb6add7c8a6183143e46c4fb58f544db57b080f Mon Sep 17 00:00:00 2001 From: Daniel Ciborowski Date: Wed, 15 Jan 2020 16:46:11 -0500 Subject: [PATCH 02/20] Update papermill.yml --- .ci/steps/papermill.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.ci/steps/papermill.yml b/.ci/steps/papermill.yml index 80f93c6..5d8882a 100644 --- a/.ci/steps/papermill.yml +++ b/.ci/steps/papermill.yml @@ -16,4 +16,5 @@ steps: --no-progress-bar \ -k python3 sleep 30 - displayName: '${{parameters.notebook}}' \ No newline at end of file + displayName: '${{parameters.notebook}}' + failOnStderr: true From 906ac58b4501ddf0f45df2640e73cbfc40b5f812 Mon Sep 17 00:00:00 2001 From: Daniel Ciborowski Date: Wed, 15 Jan 2020 18:59:31 -0500 Subject: [PATCH 03/20] Update azure-pipelines.yml --- .ci/azure-pipelines.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 92f8721..3df28d3 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -32,7 +32,6 @@ jobs: az login -t $(sptenent) --service-principal -u $(spidentity) --password $(spsecret) az account set --subscription $(subscriptionid) displayName: 'Build Configuration' - failOnStderr: true - bash: | source activate deployment_aml From 40dd9deda4100f4e028181520c7665fe31101252 Mon Sep 17 00:00:00 2001 From: Daniel Ciborowski Date: Fri, 17 Jan 2020 14:20:41 -0500 Subject: [PATCH 04/20] Update azure-pipelines.yml --- .ci/azure-pipelines.yml | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 3df28d3..baf9567 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -32,21 +32,25 @@ jobs: az login -t $(sptenent) --service-principal -u $(spidentity) --password $(spsecret) az account set --subscription $(subscriptionid) displayName: 'Build Configuration' - - - bash: | - source activate deployment_aml - cd {{cookiecutter.project_name}}/Keras_Tensorflow - papermill 00_AMLSetup.ipynb 00_AMLSetup_output.ipynb \ - --log-output \ - --no-progress-bar \ - -k python3 \ - -p resource_group $(azureresourcegroup) \ - -p workspace_name $(workspacename) \ - -p workspace_region $(azureregion) \ - -p subscription_id $(subscriptionid) \ - -p image_name 'modelimg' + + - task: AzureCLI@1 displayName : '00_AMLSetup.ipynb' - failOnStderr: true + inputs: + azureSubscription: $(subscriptionid) + scriptLocation: inlineScript + failOnStderr: True + inlineScript: | + source activate deployment_aml + cd {{cookiecutter.project_name}}/Keras_Tensorflow + papermill 00_AMLSetup.ipynb 00_AMLSetup_output.ipynb \ + --log-output \ + --no-progress-bar \ + -k python3 \ + -p resource_group $(azureresourcegroup) \ + -p workspace_name $(workspacename) \ + -p workspace_region $(azureregion) \ + -p subscription_id $(subscriptionid) \ + -p image_name 'modelimg' - template: steps/papermill.yml parameters: From a8cc9fa4ed22512b1150a48023e998e71f4443f3 Mon Sep 17 00:00:00 2001 From: Daniel Ciborowski Date: Fri, 17 Jan 2020 14:25:25 -0500 Subject: [PATCH 05/20] Update azure-pipelines.yml for Azure Pipelines --- .ci/azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index baf9567..af48d3d 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -38,7 +38,7 @@ jobs: inputs: azureSubscription: $(subscriptionid) scriptLocation: inlineScript - failOnStderr: True + failOnStandardError: True inlineScript: | source activate deployment_aml cd {{cookiecutter.project_name}}/Keras_Tensorflow From a7d3c799af4fc1ee56278da43e3f70bbf4477155 Mon Sep 17 00:00:00 2001 From: Daniel Ciborowski Date: Fri, 17 Jan 2020 14:26:35 -0500 Subject: [PATCH 06/20] Update environment.yml --- {{cookiecutter.project_name}}/Keras_Tensorflow/environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/{{cookiecutter.project_name}}/Keras_Tensorflow/environment.yml b/{{cookiecutter.project_name}}/Keras_Tensorflow/environment.yml index 8d9991a..3a9fcb2 100644 --- a/{{cookiecutter.project_name}}/Keras_Tensorflow/environment.yml +++ b/{{cookiecutter.project_name}}/Keras_Tensorflow/environment.yml @@ -8,6 +8,7 @@ dependencies: - cudatoolkit==9.0 - tensorflow-gpu==1.14.0 - urllib3 +- pip - pip: # Required packages for AzureML execution, history, and data preparation. - papermill==1.1.0 From 6fb20f798a9c2bdb7dbafb88d553aa2bee8cc364 Mon Sep 17 00:00:00 2001 From: Daniel Ciborowski Date: Fri, 17 Jan 2020 15:00:47 -0500 Subject: [PATCH 07/20] Update azure-pipelines.yml for Azure Pipelines --- .ci/azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index af48d3d..9394026 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -36,7 +36,7 @@ jobs: - task: AzureCLI@1 displayName : '00_AMLSetup.ipynb' inputs: - azureSubscription: $(subscriptionid) + azureSubscription: $(subscription) scriptLocation: inlineScript failOnStandardError: True inlineScript: | From 69f06bd87f902180253152b6110b0853b708025e Mon Sep 17 00:00:00 2001 From: Daniel Ciborowski Date: Fri, 17 Jan 2020 15:10:42 -0500 Subject: [PATCH 08/20] Update azure-pipelines.yml for Azure Pipelines --- .ci/azure-pipelines.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 9394026..05c45ae 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -38,7 +38,6 @@ jobs: inputs: azureSubscription: $(subscription) scriptLocation: inlineScript - failOnStandardError: True inlineScript: | source activate deployment_aml cd {{cookiecutter.project_name}}/Keras_Tensorflow From c03acaf1e36081b1aa4fffd691b52dc13c924c90 Mon Sep 17 00:00:00 2001 From: Daniel Ciborowski Date: Fri, 17 Jan 2020 15:18:47 -0500 Subject: [PATCH 09/20] Update papermill.yml --- .ci/steps/papermill.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.ci/steps/papermill.yml b/.ci/steps/papermill.yml index 5d8882a..561d823 100644 --- a/.ci/steps/papermill.yml +++ b/.ci/steps/papermill.yml @@ -4,17 +4,16 @@ parameters: notebook: 01_DataPrep.ipynb # defaults for any parameters that aren't specified location: "{{cookiecutter.project_name}}" - steps: - bash: | source activate deployment_aml export PYTHONPATH=$(pwd)/{{cookiecutter.project_name}}/Keras_Tensorflow:${PYTHONPATH} cd ${{parameters.location}} echo Execute ${{parameters.notebook}} - papermill ${{parameters.notebook}} output.ipynb \ + result=$(papermill ${{parameters.notebook}} output.ipynb \ --log-output \ --no-progress-bar \ - -k python3 + -k python3) sleep 30 + echo $result displayName: '${{parameters.notebook}}' - failOnStderr: true From 09bc6408a9727c06df8e5a9b6b028e198371ecb5 Mon Sep 17 00:00:00 2001 From: Daniel Ciborowski Date: Fri, 17 Jan 2020 15:41:37 -0500 Subject: [PATCH 10/20] Create az_papermill.yml --- .ci/steps/az_papermill.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .ci/steps/az_papermill.yml diff --git a/.ci/steps/az_papermill.yml b/.ci/steps/az_papermill.yml new file mode 100644 index 0000000..561d823 --- /dev/null +++ b/.ci/steps/az_papermill.yml @@ -0,0 +1,19 @@ + + +parameters: + notebook: 01_DataPrep.ipynb # defaults for any parameters that aren't specified + location: "{{cookiecutter.project_name}}" + +steps: +- bash: | + source activate deployment_aml + export PYTHONPATH=$(pwd)/{{cookiecutter.project_name}}/Keras_Tensorflow:${PYTHONPATH} + cd ${{parameters.location}} + echo Execute ${{parameters.notebook}} + result=$(papermill ${{parameters.notebook}} output.ipynb \ + --log-output \ + --no-progress-bar \ + -k python3) + sleep 30 + echo $result + displayName: '${{parameters.notebook}}' From 5f0795bb5b8264d4ec3b45a873780ea8c7d3bd8b Mon Sep 17 00:00:00 2001 From: Daniel Ciborowski Date: Fri, 17 Jan 2020 15:44:09 -0500 Subject: [PATCH 11/20] Update az_papermill.yml --- .ci/steps/az_papermill.yml | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/.ci/steps/az_papermill.yml b/.ci/steps/az_papermill.yml index 561d823..fa94175 100644 --- a/.ci/steps/az_papermill.yml +++ b/.ci/steps/az_papermill.yml @@ -3,17 +3,23 @@ parameters: notebook: 01_DataPrep.ipynb # defaults for any parameters that aren't specified location: "{{cookiecutter.project_name}}" + subscription: # steps: -- bash: | - source activate deployment_aml - export PYTHONPATH=$(pwd)/{{cookiecutter.project_name}}/Keras_Tensorflow:${PYTHONPATH} - cd ${{parameters.location}} - echo Execute ${{parameters.notebook}} - result=$(papermill ${{parameters.notebook}} output.ipynb \ - --log-output \ - --no-progress-bar \ - -k python3) - sleep 30 - echo $result - displayName: '${{parameters.notebook}}' + - task: AzureCLI@1 + displayName: '${{parameters.notebook}}' + inputs: + azureSubscription: ${{parameters.subscription}} + scriptLocation: inlineScript + inlineScript: | + source activate deployment_aml + export PYTHONPATH=$(pwd)/{{cookiecutter.project_name}}/Keras_Tensorflow:${PYTHONPATH} + cd ${{parameters.location}} + echo Execute ${{parameters.notebook}} + papermill ${{parameters.notebook}} output.ipynb \ + --log-output \ + --no-progress-bar \ + -k python3 + sleep 30 + echo $result + From 22cff7e683cd945bae7d165dc1f96507480da3f6 Mon Sep 17 00:00:00 2001 From: Daniel Ciborowski Date: Fri, 17 Jan 2020 15:44:26 -0500 Subject: [PATCH 12/20] Update az_papermill.yml --- .ci/steps/az_papermill.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.ci/steps/az_papermill.yml b/.ci/steps/az_papermill.yml index fa94175..0c11ac7 100644 --- a/.ci/steps/az_papermill.yml +++ b/.ci/steps/az_papermill.yml @@ -21,5 +21,3 @@ steps: --no-progress-bar \ -k python3 sleep 30 - echo $result - From aa193b5f3d9f7dc30c398d7212377cc5edd56c53 Mon Sep 17 00:00:00 2001 From: Daniel Ciborowski Date: Fri, 17 Jan 2020 15:47:17 -0500 Subject: [PATCH 13/20] Update azure-pipelines.yml --- .ci/azure-pipelines.yml | 47 ++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 05c45ae..1fadb91 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -51,45 +51,53 @@ jobs: -p subscription_id $(subscriptionid) \ -p image_name 'modelimg' - - template: steps/papermill.yml + - template: steps/az_papermill.yml parameters: notebook: 01_DevelopModel.ipynb location: "{{cookiecutter.project_name}}/Keras_Tensorflow" + subscription: $(subscription) - - template: steps/papermill.yml + - template: steps/az_papermill.yml parameters: notebook: 02_DevelopModelDriver.ipynb location: "{{cookiecutter.project_name}}/Keras_Tensorflow" + subscription: $(subscription) - - template: steps/papermill.yml + - template: steps/az_papermill.yml parameters: notebook: 03_BuildImage.ipynb location: "{{cookiecutter.project_name}}/Keras_Tensorflow" + subscription: $(subscription) - - bash: | - source activate deployment_aml - export PYTHONPATH=$(pwd)/{{cookiecutter.project_name}}/Keras_Tensorflow:${PYTHONPATH} - cd {{cookiecutter.project_name}}/Keras_Tensorflow/aks - papermill 04_DeployOnAKS.ipynb 04_DeployOnAKS_output.ipynb \ - --log-output \ - --no-progress-bar \ - -k python3 \ - -p aks_name 'aksdeployamlaks' \ - -p aks_service_name 'aksamlsvc' \ - -p aks_location $(azureregion) - sleep 1m + - task: AzureCLI@1 displayName : '04_DeployOnAKS.ipynb' - failOnStderr: true + inputs: + azureSubscription: $(subscription) + scriptLocation: inlineScript + inlineScript: | + source activate deployment_aml + export PYTHONPATH=$(pwd)/{{cookiecutter.project_name}}/Keras_Tensorflow:${PYTHONPATH} + cd {{cookiecutter.project_name}}/Keras_Tensorflow/aks + papermill 04_DeployOnAKS.ipynb 04_DeployOnAKS_output.ipynb \ + --log-output \ + --no-progress-bar \ + -k python3 \ + -p aks_name 'aksdeployamlaks' \ + -p aks_service_name 'aksamlsvc' \ + -p aks_location $(azureregion) + sleep 1m - - template: steps/papermill.yml + - template: steps/az_papermill.yml parameters: notebook: 05_TestWebApp.ipynb location: "{{cookiecutter.project_name}}/Keras_Tensorflow/aks" + subscription: $(subscription) - - template: steps/papermill.yml + - template: steps/az_papermill.yml parameters: notebook: 06_SpeedTestWebApp.ipynb location: "{{cookiecutter.project_name}}/Keras_Tensorflow/aks" + subscription: $(subscription) # - bash: | # source activate deployment_aml @@ -104,10 +112,11 @@ jobs: # -p module_name mygpumodule # displayName: '04_DeployOnIOTedge.ipynb' - - template: steps/papermill.yml + - template: steps/az_papermill.yml parameters: notebook: 07_TearDown.ipynb location: "{{cookiecutter.project_name}}/Keras_Tensorflow/aks" + subscription: $(subscription) # - template: steps/papermill.yml # parameters: From 003a132a93aa219bf349a6bd8c63eaf4d3267e5a Mon Sep 17 00:00:00 2001 From: Daniel Ciborowski Date: Fri, 17 Jan 2020 15:48:39 -0500 Subject: [PATCH 14/20] Update az_papermill.yml --- .ci/steps/az_papermill.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.ci/steps/az_papermill.yml b/.ci/steps/az_papermill.yml index 0c11ac7..aa7c6e5 100644 --- a/.ci/steps/az_papermill.yml +++ b/.ci/steps/az_papermill.yml @@ -3,7 +3,7 @@ parameters: notebook: 01_DataPrep.ipynb # defaults for any parameters that aren't specified location: "{{cookiecutter.project_name}}" - subscription: # + subscription: "" steps: - task: AzureCLI@1 @@ -12,12 +12,12 @@ steps: azureSubscription: ${{parameters.subscription}} scriptLocation: inlineScript inlineScript: | - source activate deployment_aml - export PYTHONPATH=$(pwd)/{{cookiecutter.project_name}}/Keras_Tensorflow:${PYTHONPATH} - cd ${{parameters.location}} - echo Execute ${{parameters.notebook}} - papermill ${{parameters.notebook}} output.ipynb \ - --log-output \ - --no-progress-bar \ - -k python3 - sleep 30 + source activate deployment_aml + export PYTHONPATH=$(pwd)/{{cookiecutter.project_name}}/Keras_Tensorflow:${PYTHONPATH} + cd ${{parameters.location}} + echo Execute ${{parameters.notebook}} + papermill ${{parameters.notebook}} output.ipynb \ + --log-output \ + --no-progress-bar \ + -k python3 + sleep 30 From 0cd1625cb85753f0f1b3ba78f7dbf852aeb79a92 Mon Sep 17 00:00:00 2001 From: Daniel Ciborowski Date: Fri, 17 Jan 2020 16:43:56 -0500 Subject: [PATCH 15/20] Update 00_AMLSetup.ipynb --- .../Keras_Tensorflow/00_AMLSetup.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{cookiecutter.project_name}}/Keras_Tensorflow/00_AMLSetup.ipynb b/{{cookiecutter.project_name}}/Keras_Tensorflow/00_AMLSetup.ipynb index f7654e5..0ca64b6 100644 --- a/{{cookiecutter.project_name}}/Keras_Tensorflow/00_AMLSetup.ipynb +++ b/{{cookiecutter.project_name}}/Keras_Tensorflow/00_AMLSetup.ipynb @@ -15,7 +15,7 @@ "metadata": {}, "outputs": [], "source": [ - "import azureml.core\n", + "import pyspark\n", "from azureml.core import Workspace\n", "from dotenv import set_key, get_key, find_dotenv\n", "from pathlib import Path\n", From 668235fe9057839241b2e4aa5a66266fee4dea23 Mon Sep 17 00:00:00 2001 From: Daniel Ciborowski Date: Fri, 17 Jan 2020 16:51:58 -0500 Subject: [PATCH 16/20] Update 00_AMLSetup.ipynb --- .../Keras_Tensorflow/00_AMLSetup.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{cookiecutter.project_name}}/Keras_Tensorflow/00_AMLSetup.ipynb b/{{cookiecutter.project_name}}/Keras_Tensorflow/00_AMLSetup.ipynb index 0ca64b6..f7654e5 100644 --- a/{{cookiecutter.project_name}}/Keras_Tensorflow/00_AMLSetup.ipynb +++ b/{{cookiecutter.project_name}}/Keras_Tensorflow/00_AMLSetup.ipynb @@ -15,7 +15,7 @@ "metadata": {}, "outputs": [], "source": [ - "import pyspark\n", + "import azureml.core\n", "from azureml.core import Workspace\n", "from dotenv import set_key, get_key, find_dotenv\n", "from pathlib import Path\n", From db278b96bf6582fa2b6413756c07136e0fb4582b Mon Sep 17 00:00:00 2001 From: Daniel Ciborowski Date: Fri, 17 Jan 2020 19:21:28 -0500 Subject: [PATCH 17/20] Update azure-pipelines.yml --- .ci/azure-pipelines.yml | 147 ++++++++++++++++++++-------------------- 1 file changed, 72 insertions(+), 75 deletions(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 1fadb91..a99e797 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -1,5 +1,12 @@ # Pipeline for testing AKSDeploymentTutorialAML +resources: + repositories: + - repository: aitemplates + type: github + name: microsoft/AI + endpoint: AIArchitecturesAndPractices-GitHub + trigger: batch: true branches: @@ -22,82 +29,74 @@ jobs: - script: echo Hello, AKSDeployment! displayName: 'Builds source for AKSDeploymentTutorialAML/Keras_Tensorflow' - - bash: | - cd {{cookiecutter.project_name}}/Keras_Tensorflow - conda env create -f environment.yml - source activate deployment_aml - conda env list - echo Login Azure Account - - az login -t $(sptenent) --service-principal -u $(spidentity) --password $(spsecret) - az account set --subscription $(subscriptionid) - displayName: 'Build Configuration' - - - task: AzureCLI@1 - displayName : '00_AMLSetup.ipynb' - inputs: + - template: .ci/steps/config_conda.yml@aitemplates + parameters: + conda_location: {{cookiecutter.project_name}}/Keras_Tensorflow azureSubscription: $(subscription) - scriptLocation: inlineScript - inlineScript: | - source activate deployment_aml - cd {{cookiecutter.project_name}}/Keras_Tensorflow - papermill 00_AMLSetup.ipynb 00_AMLSetup_output.ipynb \ - --log-output \ - --no-progress-bar \ - -k python3 \ - -p resource_group $(azureresourcegroup) \ - -p workspace_name $(workspacename) \ - -p workspace_region $(azureregion) \ - -p subscription_id $(subscriptionid) \ - -p image_name 'modelimg' - - - template: steps/az_papermill.yml + conda: deployment_aml + + - template: .ci/steps/azpapermill.yml@aitemplates parameters: - notebook: 01_DevelopModel.ipynb - location: "{{cookiecutter.project_name}}/Keras_Tensorflow" - subscription: $(subscription) + notebook: 00_AMLSetup.ipynb + location: {{cookiecutter.project_name}}/Keras_Tensorflow + azureSubscription: $(subscription) + conda: deployment_aml + azure_subscription: $(subscriptionid) + azureresourcegroup: $(azureresourcegroup) + workspacename: $(workspacename) + azureregion: $(azureregion) + aksimagename: 'modelimg' + + - template: .ci/steps/azpapermill.yml@aitemplates + parameters: + notebook: 01_DevelopModel.ipynb + location: {{cookiecutter.project_name}}/Keras_Tensorflow + azureSubscription: $(subscription) + conda: deployment_aml + azure_subscription: $(subscriptionid) - - template: steps/az_papermill.yml + - template: .ci/steps/azpapermill.yml@aitemplates parameters: - notebook: 02_DevelopModelDriver.ipynb - location: "{{cookiecutter.project_name}}/Keras_Tensorflow" - subscription: $(subscription) + notebook: 02_DevelopModelDriver.ipynb + location: {{cookiecutter.project_name}}/Keras_Tensorflow + azureSubscription: $(subscription) + conda: deployment_aml + azure_subscription: $(subscriptionid) - - template: steps/az_papermill.yml + - template: .ci/steps/azpapermill.yml@aitemplates parameters: - notebook: 03_BuildImage.ipynb - location: "{{cookiecutter.project_name}}/Keras_Tensorflow" - subscription: $(subscription) + notebook: 03_BuildImage.ipynb + location: {{cookiecutter.project_name}}/Keras_Tensorflow + azureSubscription: $(subscription) + conda: deployment_aml + azure_subscription: $(subscriptionid) - - task: AzureCLI@1 - displayName : '04_DeployOnAKS.ipynb' - inputs: - azureSubscription: $(subscription) - scriptLocation: inlineScript - inlineScript: | - source activate deployment_aml - export PYTHONPATH=$(pwd)/{{cookiecutter.project_name}}/Keras_Tensorflow:${PYTHONPATH} - cd {{cookiecutter.project_name}}/Keras_Tensorflow/aks - papermill 04_DeployOnAKS.ipynb 04_DeployOnAKS_output.ipynb \ - --log-output \ - --no-progress-bar \ - -k python3 \ - -p aks_name 'aksdeployamlaks' \ - -p aks_service_name 'aksamlsvc' \ - -p aks_location $(azureregion) - sleep 1m - - - template: steps/az_papermill.yml + - template: azpapermill.yml + parameters: + notebook: aks/04_DeployOnAKS.ipynb + location: {{cookiecutter.project_name}}/Keras_Tensorflow + azureSubscription: $(subscriptionid) + conda: ${{parameters.conda}} + aks_name: 'aksdeployamlaks' + aks_location: $(azureregion) + aks_service_name: 'aksamlsvc' + python_path: /{{cookiecutter.project_name}}/Keras_Tensorflow + + - template: .ci/steps/azpapermill.yml@aitemplates parameters: - notebook: 05_TestWebApp.ipynb - location: "{{cookiecutter.project_name}}/Keras_Tensorflow/aks" - subscription: $(subscription) + notebook: 05_TestWebApp.ipynb + location: {{cookiecutter.project_name}}/Keras_Tensorflow + azureSubscription: $(subscription) + conda: deployment_aml + azure_subscription: $(subscriptionid) - - template: steps/az_papermill.yml + - template: .ci/steps/azpapermill.yml@aitemplates parameters: - notebook: 06_SpeedTestWebApp.ipynb - location: "{{cookiecutter.project_name}}/Keras_Tensorflow/aks" - subscription: $(subscription) + notebook: 06_SpeedTestWebApp.ipynb + location: {{cookiecutter.project_name}}/Keras_Tensorflow + azureSubscription: $(subscription) + conda: deployment_aml + azure_subscription: $(subscriptionid) # - bash: | # source activate deployment_aml @@ -112,16 +111,14 @@ jobs: # -p module_name mygpumodule # displayName: '04_DeployOnIOTedge.ipynb' - - template: steps/az_papermill.yml + - template: .ci/steps/azpapermill.yml@aitemplates parameters: - notebook: 07_TearDown.ipynb - location: "{{cookiecutter.project_name}}/Keras_Tensorflow/aks" - subscription: $(subscription) - -# - template: steps/papermill.yml -# parameters: -# notebook: 05_TearDown.ipynb -# location: "{{cookiecutter.project_name}}/Keras_Tensorflow/iotedge" + notebook: 07_TearDown.ipynb + location: {{cookiecutter.project_name}}/Keras_Tensorflow + azureSubscription: $(subscription) + conda: deployment_aml + azure_subscription: $(subscriptionid) + - bash: | echo Remove All Docker Containers docker stop $(docker ps -a -q) From 0234a772ce7d01565b2047b6d2173fbad3c06ec3 Mon Sep 17 00:00:00 2001 From: Daniel Ciborowski Date: Fri, 17 Jan 2020 19:28:08 -0500 Subject: [PATCH 18/20] Update azure-pipelines.yml for Azure Pipelines --- .ci/azure-pipelines.yml | 87 ++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 50 deletions(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index a99e797..d632277 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -31,14 +31,14 @@ jobs: - template: .ci/steps/config_conda.yml@aitemplates parameters: - conda_location: {{cookiecutter.project_name}}/Keras_Tensorflow - azureSubscription: $(subscription) + conda_location: "{{cookiecutter.project_name}}/Keras_Tensorflow" + azureSubscription: $(subscription) conda: deployment_aml - + - template: .ci/steps/azpapermill.yml@aitemplates parameters: notebook: 00_AMLSetup.ipynb - location: {{cookiecutter.project_name}}/Keras_Tensorflow + location: "{{cookiecutter.project_name}}/Keras_Tensorflow" azureSubscription: $(subscription) conda: deployment_aml azure_subscription: $(subscriptionid) @@ -46,78 +46,64 @@ jobs: workspacename: $(workspacename) azureregion: $(azureregion) aksimagename: 'modelimg' - + - template: .ci/steps/azpapermill.yml@aitemplates parameters: - notebook: 01_DevelopModel.ipynb - location: {{cookiecutter.project_name}}/Keras_Tensorflow - azureSubscription: $(subscription) - conda: deployment_aml - azure_subscription: $(subscriptionid) + notebook: 01_DevelopModel.ipynb + location: "{{cookiecutter.project_name}}/Keras_Tensorflow" + azureSubscription: $(subscription) + conda: deployment_aml + azure_subscription: $(subscriptionid) - template: .ci/steps/azpapermill.yml@aitemplates parameters: - notebook: 02_DevelopModelDriver.ipynb - location: {{cookiecutter.project_name}}/Keras_Tensorflow - azureSubscription: $(subscription) - conda: deployment_aml - azure_subscription: $(subscriptionid) + notebook: 02_DevelopModelDriver.ipynb + location: "{{cookiecutter.project_name}}/Keras_Tensorflow" + azureSubscription: $(subscription) + conda: deployment_aml + azure_subscription: $(subscriptionid) - template: .ci/steps/azpapermill.yml@aitemplates parameters: - notebook: 03_BuildImage.ipynb - location: {{cookiecutter.project_name}}/Keras_Tensorflow - azureSubscription: $(subscription) - conda: deployment_aml - azure_subscription: $(subscriptionid) + notebook: 03_BuildImage.ipynb + location: "{{cookiecutter.project_name}}/Keras_Tensorflow" + azureSubscription: $(subscription) + conda: deployment_aml + azure_subscription: $(subscriptionid) - template: azpapermill.yml parameters: notebook: aks/04_DeployOnAKS.ipynb - location: {{cookiecutter.project_name}}/Keras_Tensorflow + location: "{{cookiecutter.project_name}}/Keras_Tensorflow" azureSubscription: $(subscriptionid) conda: ${{parameters.conda}} aks_name: 'aksdeployamlaks' aks_location: $(azureregion) aks_service_name: 'aksamlsvc' - python_path: /{{cookiecutter.project_name}}/Keras_Tensorflow + python_path: "/{{cookiecutter.project_name}}/Keras_Tensorflow" - template: .ci/steps/azpapermill.yml@aitemplates parameters: - notebook: 05_TestWebApp.ipynb - location: {{cookiecutter.project_name}}/Keras_Tensorflow - azureSubscription: $(subscription) - conda: deployment_aml - azure_subscription: $(subscriptionid) + notebook: 05_TestWebApp.ipynb + location: "{{cookiecutter.project_name}}/Keras_Tensorflow" + azureSubscription: $(subscription) + conda: deployment_aml + azure_subscription: $(subscriptionid) - template: .ci/steps/azpapermill.yml@aitemplates parameters: - notebook: 06_SpeedTestWebApp.ipynb - location: {{cookiecutter.project_name}}/Keras_Tensorflow - azureSubscription: $(subscription) - conda: deployment_aml - azure_subscription: $(subscriptionid) - -# - bash: | -# source activate deployment_aml -# export PYTHONPATH=$(pwd)/{{cookiecutter.project_name}}/Keras_Tensorflow:${PYTHONPATH} -# cd {{cookiecutter.project_name}}/Keras_Tensorflow/iotedge -# papermill 04_DeployOnIOTedge.ipynb 04_DeployOnIOTedge_output.ipynb \ -# --log-output \ -# --no-progress-bar \ -# -k python3 \ -# -p iot_hub_name fstlstnameiothub \ -# -p device_id mygpudevice \ -# -p module_name mygpumodule -# displayName: '04_DeployOnIOTedge.ipynb' + notebook: 06_SpeedTestWebApp.ipynb + location: "{{cookiecutter.project_name}}/Keras_Tensorflow" + azureSubscription: $(subscription) + conda: deployment_aml + azure_subscription: $(subscriptionid) - template: .ci/steps/azpapermill.yml@aitemplates parameters: - notebook: 07_TearDown.ipynb - location: {{cookiecutter.project_name}}/Keras_Tensorflow - azureSubscription: $(subscription) - conda: deployment_aml - azure_subscription: $(subscriptionid) + notebook: 07_TearDown.ipynb + location: "{{cookiecutter.project_name}}/Keras_Tensorflow" + azureSubscription: $(subscription) + conda: deployment_aml - bash: | echo Remove All Docker Containers @@ -157,3 +143,4 @@ jobs: Description=Branch: Branch $(Build.SourceBranch) failed to build. Go to Boards>WorkItems and tag the failure type. displayName: 'Create work item on failure' condition: failed() + From 9a26fd0e1eab3ac9e847da80a9d938af9f414507 Mon Sep 17 00:00:00 2001 From: Daniel Ciborowski Date: Fri, 17 Jan 2020 19:29:39 -0500 Subject: [PATCH 19/20] Update azure-pipelines.yml for Azure Pipelines --- .ci/azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index d632277..16c9b9f 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -45,7 +45,7 @@ jobs: azureresourcegroup: $(azureresourcegroup) workspacename: $(workspacename) azureregion: $(azureregion) - aksimagename: 'modelimg' + aksimagename: 'modelimg' - template: .ci/steps/azpapermill.yml@aitemplates parameters: @@ -71,7 +71,7 @@ jobs: conda: deployment_aml azure_subscription: $(subscriptionid) - - template: azpapermill.yml + - template: .ci/steps/azpapermill.yml@aitemplates parameters: notebook: aks/04_DeployOnAKS.ipynb location: "{{cookiecutter.project_name}}/Keras_Tensorflow" From 4aee33777b9bdc3026696609247d9fecde9274d2 Mon Sep 17 00:00:00 2001 From: Daniel Ciborowski Date: Fri, 17 Jan 2020 19:30:39 -0500 Subject: [PATCH 20/20] Update azure-pipelines.yml for Azure Pipelines --- .ci/azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 16c9b9f..2571679 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -76,7 +76,7 @@ jobs: notebook: aks/04_DeployOnAKS.ipynb location: "{{cookiecutter.project_name}}/Keras_Tensorflow" azureSubscription: $(subscriptionid) - conda: ${{parameters.conda}} + conda: deployment_aml aks_name: 'aksdeployamlaks' aks_location: $(azureregion) aks_service_name: 'aksamlsvc'