7 Pipelines avec Azure ML.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pipelines avec Azure ML"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src='https://github.com/retkowsky/images/blob/master/AzureMLservicebanniere.png?raw=true'>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> https://docs.microsoft.com/en-us/azure/machine-learning/service/concept-ml-pipelines"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Infos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'3.6.9 |Anaconda, Inc.| (default, Jul 30 2019, 19:07:31) \\n[GCC 7.3.0]'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "sys.version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-09-16 13:00:36.350209\n"
     ]
    }
   ],
   "source": [
    "import datetime\n",
    "now = datetime.datetime.now()\n",
    "print(now)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Version Azure ML : 1.13.0\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import azureml.core\n",
    "from azureml.core import Workspace, Experiment, Datastore\n",
    "from azureml.widgets import RunDetails\n",
    "\n",
    "print(\"Version Azure ML :\", azureml.core.VERSION)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Chargement config workspace\n",
    "ws = Workspace.from_config()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Données"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Uploading an estimated of 2 files\n",
      "Uploading ./donnees/diabetes.csv\n",
      "Uploaded ./donnees/diabetes.csv, 1 files out of an estimated total of 2\n",
      "Uploading ./donnees/diabetes2.csv\n",
      "Uploaded ./donnees/diabetes2.csv, 2 files out of an estimated total of 2\n",
      "Uploaded 2 files\n",
      "OK\n"
     ]
    }
   ],
   "source": [
    "from azureml.core import Dataset\n",
    "\n",
    "default_ds = ws.get_default_datastore()\n",
    "default_ds.upload_files(files=['./donnees/diabetes.csv', './donnees/diabetes2.csv'], # Upload the diabetes csv files in /data\n",
    "                       target_path='diabetes-data/', # Put it in a folder path in the datastore\n",
    "                       overwrite=True, # Replace existing files of the same name\n",
    "                       show_progress=True)\n",
    "\n",
    "#Create a tabular dataset from the path on the datastore (this may take a short while)\n",
    "tab_data_set = Dataset.Tabular.from_delimited_files(path=(default_ds, 'diabetes-data/*.csv'))\n",
    "\n",
    "# Register the tabular dataset\n",
    "tab_data_set = tab_data_set.register(workspace=ws, \n",
    "                           name='diabetes dataset',\n",
    "                           description='diabetes data',\n",
    "                           tags = {'format':'CSV'},\n",
    "                           create_new_version=True)\n",
    "\n",
    "print('OK')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Compute Targets :\n",
      "\t AutoML : AmlCompute\n",
      "\t Designer : AmlCompute\n",
      "\t notebooksjupyter : ComputeInstance\n",
      "\t cpucluster : AmlCompute\n",
      "\t AutoMLsdk : AmlCompute\n",
      "Datastores :\n",
      "\t azureml_globaldatasets : AzureBlob\n",
      "\t azureml : AzureBlob\n",
      "\t modelservingdata : AzureBlob\n",
      "\t aiexportdata : AzureBlob\n",
      "\t modeldata : AzureBlob\n",
      "\t teststorageserge : AzureBlob\n",
      "\t workspaceblobstore : AzureBlob\n",
      "\t workspacefilestore : AzureFile\n",
      "Datasets :\n",
      "\t sample_request_data\n",
      "\t diabetes dataset\n",
      "\t diabetes\n",
      "\t OJSales_Validate\n",
      "\t OJSales_Train\n",
      "\t OJSales_All\n",
      "\t titanic\n",
      "\t DefectScores\n",
      "\t MD-Defect-Train_Model-Trained_model-9b4070d1\n",
      "\t Failure\n",
      "\t MD-Maintenance-Train_Model-Trained_model-9b4070d1\n",
      "\t Maintenance\n",
      "\t Transactions\n",
      "\t NOAA-Weather-DS4\n",
      "\t drift-demo-dataset\n",
      "\t target\n",
      "\t dataset\n",
      "\t Iris\n",
      "\t mnist dataset\n"
     ]
    }
   ],
   "source": [
    "from azureml.core import ComputeTarget, Datastore, Dataset\n",
    "\n",
    "print(\"Compute Targets :\")\n",
    "for compute_name in ws.compute_targets:\n",
    "    compute = ws.compute_targets[compute_name]\n",
    "    print(\"\\t\", compute.name, ':', compute.type)\n",
    "    \n",
    "print(\"Datastores :\")\n",
    "for datastore_name in ws.datastores:\n",
    "    datastore = Datastore.get(ws, datastore_name)\n",
    "    print(\"\\t\", datastore.name, ':', datastore.datastore_type)\n",
    "    \n",
    "print(\"Datasets :\")\n",
    "for dataset_name in list(ws.datasets.keys()):\n",
    "    dataset = Dataset.get_by_name(ws, dataset_name)\n",
    "    print(\"\\t\", dataset.name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Création des scripts pour le pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "experiment_folder = 'WorkshopPipelines'\n",
    "os.makedirs(experiment_folder, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting WorkshopPipelines/train_diabetes.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile $experiment_folder/train_diabetes.py\n",
    "# Import libraries\n",
    "from azureml.core import Run\n",
    "import argparse\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import joblib\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import roc_curve\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Get parameters\n",
    "parser = argparse.ArgumentParser()\n",
    "parser.add_argument('--output_folder', type=str, dest='output_folder', default=\"diabetes_model\", help='output folder')\n",
    "args = parser.parse_args()\n",
    "output_folder = args.output_folder\n",
    "\n",
    "# Get the experiment run context\n",
    "run = Run.get_context()\n",
    "\n",
    "# load the diabetes data (passed as an input dataset)\n",
    "print(\"Chargement données...\")\n",
    "diabetes = run.input_datasets['diabetes_train'].to_pandas_dataframe()\n",
    "\n",
    "# Separate features and labels\n",
    "X, y = diabetes[['Pregnancies','PlasmaGlucose','DiastolicBloodPressure','TricepsThickness','SerumInsulin','BMI','DiabetesPedigree','Age']].values, diabetes['Diabetic'].values\n",
    "\n",
    "# Split data into training set and test set\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30, random_state=0)\n",
    "\n",
    "# Train adecision tree model\n",
    "print('Arbre de décision')\n",
    "model = DecisionTreeClassifier().fit(X_train, y_train)\n",
    "\n",
    "# calculate accuracy\n",
    "y_hat = model.predict(X_test)\n",
    "acc = np.average(y_hat == y_test)\n",
    "print('Accuracy =', acc)\n",
    "run.log('Accuracy', np.float(acc))\n",
    "\n",
    "# calculate AUC\n",
    "y_scores = model.predict_proba(X_test)\n",
    "auc = roc_auc_score(y_test,y_scores[:,1])\n",
    "print('AUC =' + str(auc))\n",
    "run.log('AUC', np.float(auc))\n",
    "\n",
    "# plot ROC curve\n",
    "fpr, tpr, thresholds = roc_curve(y_test, y_scores[:,1])\n",
    "fig = plt.figure(figsize=(6, 4))\n",
    "# Plot the diagonal 50% line\n",
    "plt.plot([0, 1], [0, 1], 'k--')\n",
    "# Plot the FPR and TPR achieved by our model\n",
    "plt.plot(fpr, tpr)\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('Courbe de ROC')\n",
    "run.log_image(name = \"ROC\", plot = fig)\n",
    "plt.show()\n",
    "\n",
    "# Save the trained model\n",
    "os.makedirs(output_folder, exist_ok=True)\n",
    "output_path = output_folder + \"/model.pkl\"\n",
    "joblib.dump(value=model, filename=output_path)\n",
    "\n",
    "run.complete()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rwxrwxrwx 1 root root 2085 Sep 16 13:00 WorkshopPipelines/train_diabetes.py\r\n"
     ]
    }
   ],
   "source": [
    "!ls WorkshopPipelines/train_diabetes.py -l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting WorkshopPipelines/register_diabetes.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile $experiment_folder/register_diabetes.py\n",
    "# Import libraries\n",
    "import argparse\n",
    "import joblib\n",
    "from azureml.core import Workspace, Model, Run\n",
    "\n",
    "# Get parameters\n",
    "parser = argparse.ArgumentParser()\n",
    "parser.add_argument('--model_folder', type=str, dest='model_folder', default=\"diabetes_model\", help='model location')\n",
    "args = parser.parse_args()\n",
    "model_folder = args.model_folder\n",
    "\n",
    "# Get the experiment run context\n",
    "run = Run.get_context()\n",
    "\n",
    "# load the model\n",
    "print(\"Chargement du modèle \" + model_folder)\n",
    "model_file = model_folder + \"/model.pkl\"\n",
    "model = joblib.load(model_file)\n",
    "\n",
    "Model.register(workspace=run.experiment.workspace,\n",
    "               model_path = model_file,\n",
    "               model_name = 'diabetes_model',\n",
    "               tags={'Training context':'Pipeline'})\n",
    "\n",
    "run.complete()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rwxrwxrwx 1 root root 730 Sep 16 13:00 WorkshopPipelines/register_diabetes.py\r\n"
     ]
    }
   ],
   "source": [
    "!ls WorkshopPipelines/register_diabetes.py -l"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Création Azure ML compute et environnement\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating\n",
      "Succeeded.................\n",
      "AmlCompute wait for completion finished\n",
      "\n",
      "Minimum number of nodes requested have been provisioned\n"
     ]
    }
   ],
   "source": [
    "from azureml.core.compute import ComputeTarget, AmlCompute\n",
    "from azureml.core.compute_target import ComputeTargetException\n",
    "\n",
    "cluster_name = \"cpupipelines\"\n",
    "\n",
    "# Verify that cluster exists\n",
    "try:\n",
    "    pipeline_cluster = ComputeTarget(workspace=ws, name=cluster_name)\n",
    "    print('Found existing cluster, use it.')\n",
    "except ComputeTargetException:\n",
    "    # If not, create it\n",
    "    compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_D4_V2', \n",
    "                                                           #vm_priority='lowpriority',\n",
    "                                                           min_nodes=1,\n",
    "                                                           max_nodes=4)\n",
    "    pipeline_cluster = ComputeTarget.create(ws, cluster_name, compute_config)\n",
    "\n",
    "pipeline_cluster.wait_for_completion(show_output=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AutoML  -  AmlCompute  -  Succeeded\n",
      "Designer  -  AmlCompute  -  Succeeded\n",
      "notebooksjupyter  -  ComputeInstance  -  Succeeded\n",
      "cpucluster  -  AmlCompute  -  Succeeded\n",
      "AutoMLsdk  -  AmlCompute  -  Succeeded\n",
      "cpupipelines  -  AmlCompute  -  Succeeded\n"
     ]
    }
   ],
   "source": [
    "compute_targets = ws.compute_targets\n",
    "for name, ct in compute_targets.items():\n",
    "    print(name, \" - \" , ct.type, \" - \", ct.provisioning_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Statut du compute server :\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'currentNodeCount': 1,\n",
       " 'targetNodeCount': 1,\n",
       " 'nodeStateCounts': {'preparingNodeCount': 1,\n",
       "  'runningNodeCount': 0,\n",
       "  'idleNodeCount': 0,\n",
       "  'unusableNodeCount': 0,\n",
       "  'leavingNodeCount': 0,\n",
       "  'preemptedNodeCount': 0},\n",
       " 'allocationState': 'Steady',\n",
       " 'allocationStateTransitionTime': '2020-09-16T13:02:35.756000+00:00',\n",
       " 'errors': None,\n",
       " 'creationTime': '2020-09-16T13:00:55.544388+00:00',\n",
       " 'modifiedTime': '2020-09-16T13:01:11.377412+00:00',\n",
       " 'provisioningState': 'Succeeded',\n",
       " 'provisioningStateTransitionTime': None,\n",
       " 'scaleSettings': {'minNodeCount': 1,\n",
       "  'maxNodeCount': 4,\n",
       "  'nodeIdleTimeBeforeScaleDown': 'PT120S'},\n",
       " 'vmPriority': 'Dedicated',\n",
       " 'vmSize': 'STANDARD_D4_V2'}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Statut du compute server :\")\n",
    "pipeline_cluster.get_status().serialize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OK!\n"
     ]
    }
   ],
   "source": [
    "from azureml.core import Environment\n",
    "from azureml.core.conda_dependencies import CondaDependencies\n",
    "from azureml.core.runconfig import RunConfiguration\n",
    "\n",
    "# Create a Python environment for the experiment\n",
    "diabetes_env = Environment(\"diabetes-experiment-env\")\n",
    "diabetes_env.python.user_managed_dependencies = False # Let Azure ML manage dependencies\n",
    "diabetes_env.docker.enabled = True # Use a docker container\n",
    "\n",
    "# Create a set of package dependencies\n",
    "diabetes_packages = CondaDependencies.create(conda_packages=['scikit-learn','ipykernel','matplotlib', 'pandas'],\n",
    "                                             pip_packages=['azureml-sdk','pyarrow', 'azureml-defaults'])\n",
    "\n",
    "# Add the dependencies to the environment\n",
    "diabetes_env.python.conda_dependencies = diabetes_packages\n",
    "\n",
    "# Register the environment (just in case previous lab wasn't completed)\n",
    "diabetes_env.register(workspace=ws)\n",
    "registered_env = Environment.get(ws, 'diabetes-experiment-env')\n",
    "\n",
    "# Create a new runconfig object for the pipeline\n",
    "pipeline_run_config = RunConfiguration()\n",
    "\n",
    "# Use the compute you created above. \n",
    "pipeline_run_config.target = pipeline_cluster\n",
    "\n",
    "# Assign the environment to the run configuration\n",
    "pipeline_run_config.environment = registered_env\n",
    "\n",
    "print (\"OK!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Création et exécution du pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OK!\n"
     ]
    }
   ],
   "source": [
    "from azureml.pipeline.core import PipelineData\n",
    "from azureml.pipeline.steps import PythonScriptStep, EstimatorStep\n",
    "from azureml.train.estimator import Estimator\n",
    "\n",
    "# Get the training dataset\n",
    "diabetes_ds = ws.datasets.get(\"diabetes dataset\")\n",
    "\n",
    "# Create a PipelineData (temporary Data Reference) for the model folder\n",
    "model_folder = PipelineData(\"model_folder\", datastore=ws.get_default_datastore())\n",
    "\n",
    "estimator = Estimator(source_directory=experiment_folder,\n",
    "                        compute_target = pipeline_cluster,\n",
    "                        environment_definition=pipeline_run_config.environment,\n",
    "                        entry_script='train_diabetes.py')\n",
    "\n",
    "train_step = EstimatorStep(name = \"1. Train Model\",\n",
    "                           estimator=estimator, \n",
    "                           estimator_entry_script_arguments=['--output_folder', model_folder],\n",
    "                           inputs=[diabetes_ds.as_named_input('diabetes_train')],\n",
    "                           outputs=[model_folder],\n",
    "                           compute_target = pipeline_cluster,\n",
    "                           allow_reuse = True)\n",
    "\n",
    "# Step 2, run the model registration script\n",
    "register_step = PythonScriptStep(name = \"2. Register Model\",\n",
    "                                source_directory = experiment_folder,\n",
    "                                script_name = \"register_diabetes.py\",\n",
    "                                arguments = ['--model_folder', model_folder],\n",
    "                                inputs=[model_folder],\n",
    "                                compute_target = pipeline_cluster,\n",
    "                                runconfig = pipeline_run_config,\n",
    "                                allow_reuse = True)\n",
    "\n",
    "print(\"OK!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Execution du pipeline\n",
    "> Prévoir 10 minutes de temps de traitement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pipeline généré.\n",
      "Created step 1. Train Model [81722293][aa2bdcd6-503c-4f62-aa2c-5d97b5e2190e], (This step will run and generate new outputs)\n",
      "Created step 2. Register Model [be470822][4321937f-773b-42ae-966d-01a32162c0c1], (This step will run and generate new outputs)\n",
      "Submitted PipelineRun ac73dee7-6068-4baf-b3f9-a17692e3745c\n",
      "Link to Azure Machine Learning Portal: https://ml.azure.com/experiments/Exemple7-Pipeline-Diabetes/runs/ac73dee7-6068-4baf-b3f9-a17692e3745c?wsid=/subscriptions/70b8f39e-8863-49f7-b6ba-34a80799550c/resourcegroups/workshopAML2020-rg/workspaces/workshopAML2020\n",
      "Exécution du pipeline\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "55ce21aaa8204eea835e4a8d758073a3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "_PipelineWidget(widget_settings={'childWidgetDisplay': 'popup', 'send_telemetry': False, 'log_level': 'INFO', …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/aml.mini.widget.v1": "{\"status\": \"Completed\", \"workbench_run_details_uri\": \"https://ml.azure.com/experiments/Exemple7-Pipeline-Diabetes/runs/ac73dee7-6068-4baf-b3f9-a17692e3745c?wsid=/subscriptions/70b8f39e-8863-49f7-b6ba-34a80799550c/resourcegroups/workshopAML2020-rg/workspaces/workshopAML2020\", \"run_id\": \"ac73dee7-6068-4baf-b3f9-a17692e3745c\", \"run_properties\": {\"run_id\": \"ac73dee7-6068-4baf-b3f9-a17692e3745c\", \"created_utc\": \"2020-09-16T13:03:07.435449Z\", \"properties\": {\"azureml.runsource\": \"azureml.PipelineRun\", \"runSource\": \"SDK\", \"runType\": \"SDK\", \"azureml.parameters\": \"{}\"}, \"tags\": {\"azureml.pipelineComponent\": \"pipelinerun\"}, \"end_time_utc\": \"2020-09-16T13:23:23.336286Z\", \"status\": \"Completed\", \"log_files\": {\"logs/azureml/executionlogs.txt\": \"https://workshopaml2027584246021.blob.core.windows.net/azureml/ExperimentRun/dcid.ac73dee7-6068-4baf-b3f9-a17692e3745c/logs/azureml/executionlogs.txt?sv=2019-02-02&sr=b&sig=5TaRRTxwbwAiCJtivWykMyRNX6B1wT1NpTchG6z9fWo%3D&st=2020-09-16T12%3A53%3A33Z&se=2020-09-16T21%3A03%3A33Z&sp=r\", \"logs/azureml/stderrlogs.txt\": \"https://workshopaml2027584246021.blob.core.windows.net/azureml/ExperimentRun/dcid.ac73dee7-6068-4baf-b3f9-a17692e3745c/logs/azureml/stderrlogs.txt?sv=2019-02-02&sr=b&sig=04WvEB5Uw8xg5%2FDugmXsA2seTr0umy8HJju2GyzvVAA%3D&st=2020-09-16T12%3A53%3A33Z&se=2020-09-16T21%3A03%3A33Z&sp=r\", \"logs/azureml/stdoutlogs.txt\": \"https://workshopaml2027584246021.blob.core.windows.net/azureml/ExperimentRun/dcid.ac73dee7-6068-4baf-b3f9-a17692e3745c/logs/azureml/stdoutlogs.txt?sv=2019-02-02&sr=b&sig=ImPVCwlhAaLPNRpZ%2BtBgtPNgulBpbOqDMDYyZXwd%2BAI%3D&st=2020-09-16T12%3A53%3A33Z&se=2020-09-16T21%3A03%3A33Z&sp=r\"}, \"log_groups\": [[\"logs/azureml/executionlogs.txt\", \"logs/azureml/stderrlogs.txt\", \"logs/azureml/stdoutlogs.txt\"]], \"run_duration\": \"0:20:15\"}, \"child_runs\": [{\"run_id\": \"33613bc0-aba1-4188-8d1f-5cdeade4c99b\", \"name\": \"1. Train Model\", \"status\": \"Finished\", \"start_time\": \"2020-09-16T13:03:51.982336Z\", \"created_time\": \"2020-09-16T13:03:23.00086Z\", \"end_time\": \"2020-09-16T13:22:11.648251Z\", \"duration\": \"0:18:48\", \"run_number\": 938, \"metric\": null, \"run_type\": \"azureml.StepRun\", \"training_percent\": null, \"created_time_dt\": \"2020-09-16T13:03:23.00086Z\", \"is_reused\": \"\"}, {\"run_id\": \"28e38a23-7919-4d3d-8b88-2319c6c7379f\", \"name\": \"2. Register Model\", \"status\": \"Finished\", \"start_time\": \"2020-09-16T13:22:32.050541Z\", \"created_time\": \"2020-09-16T13:22:17.238967Z\", \"end_time\": \"2020-09-16T13:23:06.756382Z\", \"duration\": \"0:00:49\", \"run_number\": 939, \"metric\": null, \"run_type\": \"azureml.StepRun\", \"training_percent\": null, \"created_time_dt\": \"2020-09-16T13:22:17.238967Z\", \"is_reused\": \"\"}], \"children_metrics\": {\"categories\": null, \"series\": null, \"metricName\": null}, \"run_metrics\": [], \"run_logs\": \"[2020-09-16 13:03:22Z] Submitting 1 runs, first five are: 81722293:33613bc0-aba1-4188-8d1f-5cdeade4c99b\\n[2020-09-16 13:22:16Z] Completing processing run id 33613bc0-aba1-4188-8d1f-5cdeade4c99b.\\n[2020-09-16 13:22:17Z] Submitting 1 runs, first five are: be470822:28e38a23-7919-4d3d-8b88-2319c6c7379f\\n[2020-09-16 13:23:23Z] Completing processing run id 28e38a23-7919-4d3d-8b88-2319c6c7379f.\\n\\nRun is completed.\", \"graph\": {\"datasource_nodes\": {\"c473a7b5\": {\"node_id\": \"c473a7b5\", \"name\": \"diabetes dataset\"}}, \"module_nodes\": {\"81722293\": {\"node_id\": \"81722293\", \"name\": \"1. Train Model\", \"status\": \"Finished\", \"_is_reused\": false, \"run_id\": \"33613bc0-aba1-4188-8d1f-5cdeade4c99b\"}, \"be470822\": {\"node_id\": \"be470822\", \"name\": \"2. Register Model\", \"status\": \"Finished\", \"_is_reused\": false, \"run_id\": \"28e38a23-7919-4d3d-8b88-2319c6c7379f\"}}, \"edges\": [{\"source_node_id\": \"c473a7b5\", \"source_node_name\": \"diabetes dataset\", \"source_name\": \"data\", \"target_name\": \"diabetes_train\", \"dst_node_id\": \"81722293\", \"dst_node_name\": \"1. Train Model\"}, {\"source_node_id\": \"81722293\", \"source_node_name\": \"1. Train Model\", \"source_name\": \"model_folder\", \"target_name\": \"model_folder\", \"dst_node_id\": \"be470822\", \"dst_node_name\": \"2. Register Model\"}], \"child_runs\": [{\"run_id\": \"33613bc0-aba1-4188-8d1f-5cdeade4c99b\", \"name\": \"1. Train Model\", \"status\": \"Finished\", \"start_time\": \"2020-09-16T13:03:51.982336Z\", \"created_time\": \"2020-09-16T13:03:23.00086Z\", \"end_time\": \"2020-09-16T13:22:11.648251Z\", \"duration\": \"0:18:48\", \"run_number\": 938, \"metric\": null, \"run_type\": \"azureml.StepRun\", \"training_percent\": null, \"created_time_dt\": \"2020-09-16T13:03:23.00086Z\", \"is_reused\": \"\"}, {\"run_id\": \"28e38a23-7919-4d3d-8b88-2319c6c7379f\", \"name\": \"2. Register Model\", \"status\": \"Finished\", \"start_time\": \"2020-09-16T13:22:32.050541Z\", \"created_time\": \"2020-09-16T13:22:17.238967Z\", \"end_time\": \"2020-09-16T13:23:06.756382Z\", \"duration\": \"0:00:49\", \"run_number\": 939, \"metric\": null, \"run_type\": \"azureml.StepRun\", \"training_percent\": null, \"created_time_dt\": \"2020-09-16T13:22:17.238967Z\", \"is_reused\": \"\"}]}, \"widget_settings\": {\"childWidgetDisplay\": \"popup\", \"send_telemetry\": false, \"log_level\": \"INFO\", \"sdk_version\": \"1.13.0\"}, \"loading\": false}"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from azureml.core import Experiment\n",
    "from azureml.pipeline.core import Pipeline\n",
    "from azureml.widgets import RunDetails\n",
    "\n",
    "# Construct the pipeline\n",
    "pipeline_steps = [train_step, register_step]\n",
    "pipeline = Pipeline(workspace = ws, steps=pipeline_steps)\n",
    "print(\"Pipeline généré.\")\n",
    "\n",
    "# Create an experiment and run the pipeline\n",
    "experiment = Experiment(workspace = ws, name = 'Exemple7-Pipeline-Diabetes')\n",
    "\n",
    "pipeline_run = experiment.submit(pipeline, regenerate_outputs=True)\n",
    "print(\"Exécution du pipeline\")\n",
    "\n",
    "RunDetails(pipeline_run).show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Progression du pipeline :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Etape du pipeline : 2. Register Model - Statut = Finished\n",
      "Etape du pipeline : 1. Train Model - Statut = Finished\n"
     ]
    }
   ],
   "source": [
    "# Pour connaitre le statut du run du pipeline\n",
    "\n",
    "step_runs = pipeline_run.get_children()\n",
    "for step_run in step_runs:\n",
    "    status = step_run.get_status()\n",
    "    print('Etape du pipeline :', step_run.name, '- Statut =', status)\n",
    "    \n",
    "    if status == \"Failed\":\n",
    "        joblog = step_run.get_job_log()\n",
    "        print('job log:', joblog)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. Publication du pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING - Retrying (Retry(total=2, connect=3, read=2, redirect=None, status=None)) after connection broken by 'ReadTimeoutError(\"HTTPSConnectionPool(host='westeurope.api.azureml.ms', port=443): Read timed out. (read timeout=100)\",)': /pipelines/v1.0/subscriptions/70b8f39e-8863-49f7-b6ba-34a80799550c/resourceGroups/workshopAML2020-rg/providers/Microsoft.MachineLearningServices/workspaces/workshopAML2020/Pipelines/Create\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table style=\"width:100%\"><tr><th>Name</th><th>Id</th><th>Status</th><th>Endpoint</th></tr><tr><td>Exemple7-Training-Pipeline</td><td><a href=\"https://ml.azure.com/pipelines/09b07455-b156-474f-9fca-0416aa0de511?wsid=/subscriptions/70b8f39e-8863-49f7-b6ba-34a80799550c/resourcegroups/workshopAML2020-rg/workspaces/workshopAML2020\" target=\"_blank\" rel=\"noopener\">09b07455-b156-474f-9fca-0416aa0de511</a></td><td>Active</td><td><a href=\"https://westeurope.api.azureml.ms/pipelines/v1.0/subscriptions/70b8f39e-8863-49f7-b6ba-34a80799550c/resourceGroups/workshopAML2020-rg/providers/Microsoft.MachineLearningServices/workspaces/workshopAML2020/PipelineRuns/PipelineSubmit/09b07455-b156-474f-9fca-0416aa0de511\" target=\"_blank\" rel=\"noopener\">REST Endpoint</a></td></tr></table>"
      ],
      "text/plain": [
       "Pipeline(Name: Exemple7-Training-Pipeline,\n",
       "Id: 09b07455-b156-474f-9fca-0416aa0de511,\n",
       "Status: Active,\n",
       "Endpoint: https://westeurope.api.azureml.ms/pipelines/v1.0/subscriptions/70b8f39e-8863-49f7-b6ba-34a80799550c/resourceGroups/workshopAML2020-rg/providers/Microsoft.MachineLearningServices/workspaces/workshopAML2020/PipelineRuns/PipelineSubmit/09b07455-b156-474f-9fca-0416aa0de511)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "experiment_name = 'Exemple7-Pipeline-Diabetes'\n",
    "\n",
    "pipeline_experiment = ws.experiments.get(experiment_name)\n",
    "pipeline_run = list(pipeline_experiment.get_runs())[0]\n",
    "\n",
    "# Publication du pipeline\n",
    "published_pipeline = pipeline_run.publish_pipeline(name=\"Exemple7-Training-Pipeline\", description=\"Pipeline Diabetes\", version=\"1.0\")\n",
    "\n",
    "published_pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### API du pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Endpoint du pipeline :\n",
      "https://westeurope.api.azureml.ms/pipelines/v1.0/subscriptions/70b8f39e-8863-49f7-b6ba-34a80799550c/resourceGroups/workshopAML2020-rg/providers/Microsoft.MachineLearningServices/workspaces/workshopAML2020/PipelineRuns/PipelineSubmit/09b07455-b156-474f-9fca-0416aa0de511\n"
     ]
    }
   ],
   "source": [
    "rest_endpoint = published_pipeline.endpoint\n",
    "print(\"Endpoint du pipeline :\")\n",
    "print(rest_endpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Authorization': 'Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6ImppYk5ia0ZTU2JteFBZck45Q0ZxUms0SzRndyIsImtpZCI6ImppYk5ia0ZTU2JteFBZck45Q0ZxUms0SzRndyJ9.eyJhdWQiOiJodHRwczovL21hbmFnZW1lbnQuY29yZS53aW5kb3dzLm5ldC8iLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC83MmY5ODhiZi04NmYxLTQxYWYtOTFhYi0yZDdjZDAxMWRiNDcvIiwiaWF0IjoxNjAwMjYwODU5LCJuYmYiOjE2MDAyNjA4NTksImV4cCI6MTYwMDI2NDc1OSwiYWNyIjoiMSIsImFpbyI6IkFWUUFxLzhRQUFBQXVBQjFoLzVpZUp6K1RWUUM3YWw5VEs4Q21GQlpGZGxvVUNQS3M0MlJSLzA3ZytYUzNwaEdmeWhNdkxUSVFZUldyNUd4blhnNHFQSTRPOWFsYlp6c0tPUVUycUsxNExCS3NYTnZzU2MzcUFJPSIsImFtciI6WyJyc2EiLCJtZmEiXSwiYXBwaWQiOiIwNGIwNzc5NS04ZGRiLTQ2MWEtYmJlZS0wMmY5ZTFiZjdiNDYiLCJhcHBpZGFjciI6IjAiLCJmYW1pbHlfbmFtZSI6IlJldGtvd3NreSIsImdpdmVuX25hbWUiOiJTZXJnZSIsImdyb3VwcyI6WyIyNDdjZTQwMC1iNGExLTQwODMtYjcwMC01N2YzZDdmYjcyZTQiLCIzYWE1M2ZlYi00OTY2LTQ5YTQtODk1Ny0xZDliOWFmNjVlMjUiLCI2NzE0ZjczMy0wNjVlLTQ3ZjctYmZjNy05OTZkNWQyYjYwOGMiLCJmMzg1NGE5MS1hMTc2LTRmOTEtODY4OS1kMWViNTkyZDlkYTYiLCJhN2UzMTZlNS02YjVkLTQ3YmUtYWZiOC1iN2Q2YjQyZWIyMTQiLCIyMmNjMmE1ZS04N2ZhLTQyOTQtYjdiNS0wN2JiZTc1M2VjZTIiLCIwZDRiMTRjMC1iYTc2LTRhOTAtOWIyYS1iZmQzM2U4M2JiMGIiLCI4YjMzYTYwOC00ZmViLTRkMmQtOWYyZC0zMTExNTVmOTA4M2MiLCI2NzkzOWRlNy0zZDE3LTQ5YzQtOTZhNy0xNTYxZWZhMjUxNGUiLCJlNjEzOGVlMi1iNTY3LTQ5ZWItOTdkZi04ZjE3Nzc2ODJhY2YiLCI0MDgwZmFmOS0zMTY0LTQ3ZmUtODMxNy00ZDI4MTEwMTM5M2QiLCJkMmEyNmM5MS02MmNhLTRhZWUtOGU3My0zZmM5NWJiMDU0NTciLCJhNTBmNGE4NC1jZmY4LTQyM2MtYTVjNy1jODhmYjRjNGJkZDgiLCIzNTNhZmMyYS05M2MxLTRhZWMtYThiNS0xZDVhMDIyMjJiMTgiLCI5YTMyZDJkYy00OTM2LTQ4M2YtYmExNi0yM2ViODU3YTlkNmEiLCI4MDI3M2UzNC0wNzcxLTQ5NzAtOTA4ZC1iNTQ1YWY0MDA0NzAiLCI4NTYxZmJmNy03Y2VjLTQ3ZTAtOWJhOC1jZTcyODAzMWU1ZTkiLCI2Yzk3NjRjMS1mYWUxLTQyOTYtOTE0NC1mZjM5MDg1NjE5NjQiLCJlM2U4ZDQ1ZC1hY2EzLTQ1MmQtOTJlNy04ODgzMjdhNWZiNWIiLCJlZTVkOTVmOS1mN2Y4LTQ2NjktYTE4ZS0yODAyYjBlZDNhMzYiLCIyMjY2YzhkZC0yMTJmLTRhMDEtODg4Ni1jZTUyNWE5NDY4MDQiLCJhYjFkNjEzMS0yNTE4LTQ0NTgtOTMzOS1kYTY1MDI4NDVhNjQiLCI2ZTc5NWM5My02Yzc4LTQwMDktYWIyOC1lYTAyNWZkMGY0YWYiLCI0YWJkYjYyMC1mNzE2LTQ0NWUtOTUwMy0wYzE1YTNjZWMxZDEiLCJhOWZhOTI1OC1lYzdlLTQ3ODMtOGZhMi0xZjc5NjY5YWFlNDQiLCIwMjMyMGFjYS0zODc3LTQwYTAtOTQxZC1lNmEyOTMyODVhYTIiLCIwNTg3Y2JiOS0xOGViLTQ3YjQtYmFlOC01MzdhMjFlYjNjNjYiLCIwZDZkMGRiNS03ZDQxLTRlMjMtOTk0Yy1iZTM1YTNmMDc2MjgiLCJjMmZjOTNhYi02ZmNjLTQ3ZjItYTM5OS1lMzZlYzVhNmIwM2UiLCJkZTFiYTJhNC0wZjE1LTQ0NjUtYTBiMi02MGZkMzRjOTE3ZjciLCJiMzAyYWY5My1lNjRhLTQ4MmYtYmVmNi1hMzYyYWFkM2M2ZmEiLCI5ZjZmMWFmMC02NTMxLTQ0YjgtYTQ3Yy1iZDBhOWU4NmJlZjIiLCI5ZjM4MjA4OS1lOWMxLTRjOWQtOWY2OS00YzVhN2JhYzU2M2IiLCJhM2UxM2U5ZC01M2NiLTRhMTctYjdhNi1kNWFiNDVhYTA4N2IiLCJkNTkxODJmOC0xODA3LTQxNTQtYTBjNC01MmVkZmZhNGNiZjQiLCI0YmM2Njg4NC1mYWE1LTQ3M2YtOTRjMy01YTJhZDNjNjM3MDUiLCIxNzFkNWQ2NC05ZDFmLTRjZTEtOTQ4MC04YWUwZTEyYzY2MDUiLCI2Y2JlN2U1Zi1iY2YyLTRlYjAtODQzMi05N2ViOTI0MWQ1NTEiLCIwNGRlYWJkNi01NTJmLTQzNjctOTJhNS1iZGZkZDM3NTBmYWIiLCIxOGU1ODNkMy1kMmYxLTRlMTAtYTBlYi02Y2MxZjNiNzQwNTQiLCI1ZmNjMTUyNS0wZGMzLTRjZWEtODU2Ni0xODE4MDk4MmE4ZmYiLCI4YTM5YWMyYS1hYzJhLTQ5M2ItYTlkOC00Njc4NzU3MzJlZWYiLCIzMTgwY2VmYS05MzgzLTRhZWYtODE5ZS0wODcyY2QyYTcxMzAiLCI5M2ExZjBmMC1lZDczLTQwMjAtYWEyMy1kZTVmMmQ2NmQ1OTgiLCIyMmEyMjliZC1jN2EyLTQ5ZDAtOWVhYS1lMWZjODg4ZGFhYzYiLCJkZjRlZGExNS02YmY3LTQzZGMtOGRmZi03Zjg4ZTA0YzE3NDUiLCJhZTNmNDljZS1lNTQ1LTQ5NDItYmUyNi01NzdjZTlmMjc0OGIiLCIzOTlhMjRhNS1jYzg4LTQ4YjUtYmYxMC03ZmI5YzE2NGUyMTIiLCI5ZDUyOWQ1ZS1iODgyLTRmZmItYThlNi1jNDE0N2M2ODA3MzUiLCJiZThjYTM3OC1iYzc0LTQ2YzEtYjkyMi1lN2Y1NTI0ODZlZGUiLCIzNGZmNzQxOC1mMzJkLTRiMTUtYmNkMC1iYjRlYzc5Nzk2YzEiLCI3YWRkNGIwMS04YjlmLTRlOWUtYmFiOS1kZTNmZThiOTJmMzYiLCJjMDQ5YThkNy0xM2UyLTQxNTQtODM2Ni0xZjEwNDFkZDcwYWMiLCJlN2M0M2QwMC00MjAwLTQ0ZmUtOGJkYi1kNDhmYzQ4NDFlMmEiLCIwZWY5MTU3Ny02NjM3LTQ4NTAtYTRjNC04OTFhMWRmYjRhYTQiLCI2MDNlYmQ2ZS1kNmZkLTQ5MzYtOTYxYy05ZmI4NTQ1YzUwNDQiLCIwNTFiYmFkMC0xOGZlLTQwMGEtODc1ZS0xMWNmZmYxZTQ4ZjUiLCI3ZTFiZGUxZS0zMWY5LTQwMmEtOWMwYy05NzZhMzUwZWIyOTYiLCI1NWVmYTljOS05MjQ2LTQ4YmYtOTdhNC05OTg3OTc3YzRkNWQiLCJiNTE1NDc5Yy05ZGY3LTQ0YjQtOTMwNC1jMjViMWVhOGQzNTgiLCI1MGVjNDYxMS1mODM1LTRhYTEtOGMyNS1jYzBhMGZmOTI0MzMiLCI2NjQyMzE5OS1kOTk3LTQyNDMtOTAwMC01ZjY0NWIyNzNjNzIiLCJjODMwYmU1YS03ODU1LTQwM2UtYjAxOC1lOTQ1Nzg2ZDY2ZTQiLCI4ZjQwYWY1NS01NjE3LTQzZDAtODA1Mi0yNDQ4OGVkZDlkNjYiLCIzNmM1MjZhZC1jNDhmLTQ2NGQtYTY4Ny1iNTA4ZDMyMWFkYjciLCJiZjUzMDRjZi03NzFjLTQyY2MtOWZmOC01NzdlOTlkNjYxMDEiLCJkYzFkOTcxZC04MjM5LTRjZjUtYjA5ZC1hY2FlOGFlYWZiZmUiLCIxOTI3NDFiYi03ZjRiLTRmYTItYTRjOS0xODhiMmE3MTVmZDQiLCI2MmVkYmQ3Yi04ZDQ2LTRkMmMtYTVhMS1kYTViNzhiYTFkMzgiLCJkYTBhMDFhMi1hM2M0LTQ5NmItYmM0Zi0xYzQ5MGRjOTQ3OTQiLCJkNzI5ZjhkZi1iNjMxLTQ3NWQtYWNiNS0xYTE4YTA2ODkwNWIiLCJlZjQ2Zjc0Ny1mOTk5LTRmZjAtYTY1Zi1hN2ZmNWIxMjI2NjIiXSwiaW5fY29ycCI6InRydWUiLCJpcGFkZHIiOiI1MS4xMDUuMjM2LjIyNCIsIm5hbWUiOiJTZXJnZSBSZXRrb3dza3kiLCJvaWQiOiJmZjQxMzdhOC0yZDM3LTRlZTctODY3NS05ODc2ZWJhNDY5Y2UiLCJvbnByZW1fc2lkIjoiUy0xLTUtMjEtMTcyMTI1NDc2My00NjI2OTU4MDYtMTUzODg4MjI4MS00MDYzODc5IiwicHVpZCI6IjEwMDMyMDAwMjY3NzhBNTYiLCJyaCI6IjAuQVJvQXY0ajVjdkdHcjBHUnF5MTgwQkhiUjVWM3NBVGJqUnBHdS00Qy1lR19lMFlhQUJzLiIsInNjcCI6InVzZXJfaW1wZXJzb25hdGlvbiIsInN1YiI6ImxpRk8tZTZzRXdvalJaXzJIaVpkQlhYem9wMVpseHpDbDR2ekhyMWk0UmciLCJ0aWQiOiI3MmY5ODhiZi04NmYxLTQxYWYtOTFhYi0yZDdjZDAxMWRiNDciLCJ1bmlxdWVfbmFtZSI6InNlcmV0a293QG1pY3Jvc29mdC5jb20iLCJ1cG4iOiJzZXJldGtvd0BtaWNyb3NvZnQuY29tIiwidXRpIjoiUVpWV3JCc0pzRVd0MWRUOVVwY2FBQSIsInZlciI6IjEuMCIsIndpZHMiOlsiYjc5ZmJmNGQtM2VmOS00Njg5LTgxNDMtNzZiMTk0ZTg1NTA5Il0sInhtc190Y2R0IjoxMjg5MjQxNTQ3fQ.d27EA-fx8CMJTwNlyhCtLmzyJD_B5cGh0rc1gJ2L8ISMz4uDmI8jQQV8HZpbFGPHE2u7czILwF1u8fyyOEKnPFO3q2taxtT0qaAnReu4-mf7xQcktXQ_lpgDbI2SDR-Iv05Is30beBlD1bl9eOTNrARYPQjIv0SuYEV8RNB37eQXMQAHI4gBhfaWxmBGx3C9gNGjufRFlkLbnbiiBwHQ0-EMqMN-bHwwHoT3P7Pcoo9mg2r-iQOyYl3Z4D3ztGjtc--myQzCy3cAuHR1lLAsFDam-4EgDcCksI4tEpCVyEWdKzigi0UhbYFCuOTZPQtIU8nXnx9au7SH4Oq56_8J_w'}\n"
     ]
    }
   ],
   "source": [
    "from azureml.core.authentication import InteractiveLoginAuthentication\n",
    "\n",
    "interactive_auth = InteractiveLoginAuthentication()\n",
    "auth_header = interactive_auth.get_authentication_header()\n",
    "print(auth_header)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run ID du pipeline : 6bbe06bd-9b82-42aa-b044-6d56254c9dd0\n"
     ]
    }
   ],
   "source": [
    "# Run Id du pipeline\n",
    "import requests\n",
    "\n",
    "rest_endpoint = published_pipeline.endpoint\n",
    "response = requests.post(rest_endpoint, \n",
    "                         headers=auth_header, \n",
    "                         json={\"ExperimentName\": experiment_name})\n",
    "run_id = response.json()[\"Id\"]\n",
    "print('Run ID du pipeline :', run_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7384d469530c4b81a32789513adc0e0f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "_PipelineWidget(widget_settings={'childWidgetDisplay': 'popup', 'send_telemetry': False, 'log_level': 'INFO', …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/aml.mini.widget.v1": "{\"status\": \"Completed\", \"workbench_run_details_uri\": \"https://ml.azure.com/experiments/Exemple7-Pipeline-Diabetes/runs/6bbe06bd-9b82-42aa-b044-6d56254c9dd0?wsid=/subscriptions/70b8f39e-8863-49f7-b6ba-34a80799550c/resourcegroups/workshopAML2020-rg/workspaces/workshopAML2020\", \"run_id\": \"6bbe06bd-9b82-42aa-b044-6d56254c9dd0\", \"run_properties\": {\"run_id\": \"6bbe06bd-9b82-42aa-b044-6d56254c9dd0\", \"created_utc\": \"2020-09-16T13:35:06.865057Z\", \"properties\": {\"azureml.runsource\": \"azureml.PipelineRun\", \"runSource\": \"Unavailable\", \"runType\": \"HTTP\", \"azureml.parameters\": \"{}\", \"azureml.pipelineid\": \"09b07455-b156-474f-9fca-0416aa0de511\"}, \"tags\": {\"azureml.pipelineid\": \"09b07455-b156-474f-9fca-0416aa0de511\", \"azureml.pipelineComponent\": \"pipelinerun\"}, \"end_time_utc\": \"2020-09-16T13:35:28.631687Z\", \"status\": \"Completed\", \"log_files\": {\"logs/azureml/executionlogs.txt\": \"https://workshopaml2027584246021.blob.core.windows.net/azureml/ExperimentRun/dcid.6bbe06bd-9b82-42aa-b044-6d56254c9dd0/logs/azureml/executionlogs.txt?sv=2019-02-02&sr=b&sig=iUW79TtEXyAmrA%2Bs1hfUOymcPsaka9bPpXZuSaeyVtY%3D&st=2020-09-16T13%3A25%3A37Z&se=2020-09-16T21%3A35%3A37Z&sp=r\", \"logs/azureml/stderrlogs.txt\": \"https://workshopaml2027584246021.blob.core.windows.net/azureml/ExperimentRun/dcid.6bbe06bd-9b82-42aa-b044-6d56254c9dd0/logs/azureml/stderrlogs.txt?sv=2019-02-02&sr=b&sig=JUPuEa2wuFsN3OPX2F0NPoum4aK8ysWiQq5kruFIDt0%3D&st=2020-09-16T13%3A25%3A37Z&se=2020-09-16T21%3A35%3A37Z&sp=r\", \"logs/azureml/stdoutlogs.txt\": \"https://workshopaml2027584246021.blob.core.windows.net/azureml/ExperimentRun/dcid.6bbe06bd-9b82-42aa-b044-6d56254c9dd0/logs/azureml/stdoutlogs.txt?sv=2019-02-02&sr=b&sig=jT7AN%2FpA7IY1cH7GpmbIJSKFhxv2pXpXavAXz6toUiI%3D&st=2020-09-16T13%3A25%3A37Z&se=2020-09-16T21%3A35%3A37Z&sp=r\"}, \"log_groups\": [[\"logs/azureml/executionlogs.txt\", \"logs/azureml/stderrlogs.txt\", \"logs/azureml/stdoutlogs.txt\"]], \"run_duration\": \"0:00:21\"}, \"child_runs\": [{\"run_id\": \"3470b203-b767-4a83-946a-ffd671d0d2be\", \"name\": \"1. Train Model\", \"status\": \"Finished\", \"start_time\": \"2020-09-16T13:35:24.238109Z\", \"created_time\": \"2020-09-16T13:35:24.238109Z\", \"end_time\": \"2020-09-16T13:35:24.45636Z\", \"duration\": \"0:00:00\", \"run_number\": 942, \"metric\": null, \"run_type\": \"azureml.StepRun\", \"training_percent\": null, \"created_time_dt\": \"2020-09-16T13:35:24.238109Z\", \"is_reused\": \"Yes\"}, {\"run_id\": \"4610893e-66d5-40d8-897c-ddc9fc0d5d65\", \"name\": \"2. Register Model\", \"status\": \"Finished\", \"start_time\": \"2020-09-16T13:35:28.308413Z\", \"created_time\": \"2020-09-16T13:35:28.308413Z\", \"end_time\": \"2020-09-16T13:35:28.392684Z\", \"duration\": \"0:00:00\", \"run_number\": 943, \"metric\": null, \"run_type\": \"azureml.StepRun\", \"training_percent\": null, \"created_time_dt\": \"2020-09-16T13:35:28.308413Z\", \"is_reused\": \"Yes\"}], \"children_metrics\": {\"categories\": null, \"series\": null, \"metricName\": null}, \"run_metrics\": [], \"run_logs\": \"[2020-09-16 13:35:24Z] Completing processing run id 3470b203-b767-4a83-946a-ffd671d0d2be.\\n[2020-09-16 13:35:28Z] Completing processing run id 4610893e-66d5-40d8-897c-ddc9fc0d5d65.\\n[2020-09-16 13:35:28Z] Finishing experiment: no runs left and nothing to schedule.\\n\\nRun is completed.\", \"graph\": {\"datasource_nodes\": {\"c473a7b5\": {\"node_id\": \"c473a7b5\", \"name\": \"diabetes dataset\"}}, \"module_nodes\": {\"81722293\": {\"node_id\": \"81722293\", \"name\": \"1. Train Model\", \"status\": \"Finished\", \"_is_reused\": true, \"run_id\": \"3470b203-b767-4a83-946a-ffd671d0d2be\"}, \"be470822\": {\"node_id\": \"be470822\", \"name\": \"2. Register Model\", \"status\": \"Finished\", \"_is_reused\": true, \"run_id\": \"4610893e-66d5-40d8-897c-ddc9fc0d5d65\"}}, \"edges\": [{\"source_node_id\": \"c473a7b5\", \"source_node_name\": \"diabetes dataset\", \"source_name\": \"data\", \"target_name\": \"diabetes_train\", \"dst_node_id\": \"81722293\", \"dst_node_name\": \"1. Train Model\"}, {\"source_node_id\": \"81722293\", \"source_node_name\": \"1. Train Model\", \"source_name\": \"model_folder\", \"target_name\": \"model_folder\", \"dst_node_id\": \"be470822\", \"dst_node_name\": \"2. Register Model\"}], \"child_runs\": [{\"run_id\": \"3470b203-b767-4a83-946a-ffd671d0d2be\", \"name\": \"1. Train Model\", \"status\": \"Finished\", \"start_time\": \"2020-09-16T13:35:24.238109Z\", \"created_time\": \"2020-09-16T13:35:24.238109Z\", \"end_time\": \"2020-09-16T13:35:24.45636Z\", \"duration\": \"0:00:00\", \"run_number\": 942, \"metric\": null, \"run_type\": \"azureml.StepRun\", \"training_percent\": null, \"created_time_dt\": \"2020-09-16T13:35:24.238109Z\", \"is_reused\": \"Yes\"}, {\"run_id\": \"4610893e-66d5-40d8-897c-ddc9fc0d5d65\", \"name\": \"2. Register Model\", \"status\": \"Finished\", \"start_time\": \"2020-09-16T13:35:28.308413Z\", \"created_time\": \"2020-09-16T13:35:28.308413Z\", \"end_time\": \"2020-09-16T13:35:28.392684Z\", \"duration\": \"0:00:00\", \"run_number\": 943, \"metric\": null, \"run_type\": \"azureml.StepRun\", \"training_percent\": null, \"created_time_dt\": \"2020-09-16T13:35:28.308413Z\", \"is_reused\": \"Yes\"}]}, \"widget_settings\": {\"childWidgetDisplay\": \"popup\", \"send_telemetry\": false, \"log_level\": \"INFO\", \"sdk_version\": \"1.13.0\"}, \"loading\": false}"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from azureml.pipeline.core.run import PipelineRun\n",
    "from azureml.widgets import RunDetails\n",
    "\n",
    "published_pipeline_run = PipelineRun(ws.experiments[experiment_name], run_id)\n",
    "RunDetails(published_pipeline_run).show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Planification du pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.pipeline.core import ScheduleRecurrence, Schedule\n",
    "\n",
    "# Planification du pipeline tous les lundis à 00:00 UTC\n",
    "recurrence = ScheduleRecurrence(frequency=\"Week\", interval=1, week_days=[\"Monday\"], time_of_day=\"00:00\")\n",
    "weekly_schedule = Schedule.create(ws, name=\"Pipeline-Hebdomadaire\", \n",
    "                                  description=\"Pipeline hebdomadaire Diabetes\",\n",
    "                                  pipeline_id=published_pipeline.id, \n",
    "                                  experiment_name=experiment_name, \n",
    "                                  recurrence=recurrence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Pipeline(Name: Pipeline-Hebdomadaire,\n",
       " Id: 12201367-cc07-44dc-aac5-5a909f6844f8,\n",
       " Status: Active,\n",
       " Pipeline Id: 09b07455-b156-474f-9fca-0416aa0de511,\n",
       " Recurrence Details: Runs at 0:00 on Monday every Week),\n",
       " Pipeline(Name: Pipeline-Hebdomadaire,\n",
       " Id: 64972d95-9642-405e-badc-3da3297316a5,\n",
       " Status: Active,\n",
       " Pipeline Id: 9edda14b-1ede-4636-a85a-0dc84fdb2ccd,\n",
       " Recurrence Details: Runs at 0:00 on Monday every Week),\n",
       " Pipeline(Name: Pipeline-Hebdomadaire,\n",
       " Id: 36fe78a4-4fbc-4178-8823-89166c4e382b,\n",
       " Status: Active,\n",
       " Pipeline Id: 1dc840f7-ab96-4c02-adee-df88f5a50ebd,\n",
       " Recurrence Details: Runs at 0:00 on Monday every Week),\n",
       " Pipeline(Name: Pipeline-Hebdomadaire,\n",
       " Id: 8608e3ac-0806-4b08-ab30-09a5da24602b,\n",
       " Status: Active,\n",
       " Pipeline Id: c8c34fc4-45c6-4f5c-9183-de5adb73939b,\n",
       " Recurrence Details: Runs at 0:00 on Monday every Week),\n",
       " Pipeline(Name: Pipeline-Hebdomadaire,\n",
       " Id: ab1f183f-aee4-4dc0-bbe4-439c4d3c9d47,\n",
       " Status: Active,\n",
       " Pipeline Id: 7afca3e9-2ae1-4d9a-8693-23b9461ea7bf,\n",
       " Recurrence Details: Runs at 0:00 on Monday every Week),\n",
       " Pipeline(Name: Pipeline-Hebdomadaire,\n",
       " Id: b98c561a-f67e-4848-bc0b-21a1b97d0c5d,\n",
       " Status: Active,\n",
       " Pipeline Id: 875916e7-b6e3-4a52-a38e-943820aadb20,\n",
       " Recurrence Details: Runs at 0:00 on Monday every Week),\n",
       " Pipeline(Name: RetrainingSchedule-DataIngestion,\n",
       " Id: bfdb1324-c40f-49f2-a7fd-3fae8bd75acf,\n",
       " Status: Active,\n",
       " Pipeline Id: 0efe56f0-6870-4418-8dcb-53b6d8fd4361,\n",
       " Datastore: workspaceblobstore),\n",
       " Pipeline(Name: RetrainingSchedule,\n",
       " Id: 56e22a62-1ecd-47c6-917d-182ede816934,\n",
       " Status: Active,\n",
       " Pipeline Id: 511c7b3b-cc7f-4b0d-a8e7-11f8e72c1723,\n",
       " Datastore: workspaceblobstore),\n",
       " Pipeline(Name: Pipeline-Hebdomadaire,\n",
       " Id: 785bd9dc-ba67-4aca-b298-591c111215bd,\n",
       " Status: Active,\n",
       " Pipeline Id: 2b08d3fb-1026-405e-a3c8-97ef95d35e59,\n",
       " Recurrence Details: Runs at 0:00 on Monday every Week),\n",
       " Pipeline(Name: Pipeline-Hebdomadaire,\n",
       " Id: 96ab7d54-d084-4c90-9462-f6ec684ff3f4,\n",
       " Status: Active,\n",
       " Pipeline Id: bd529cae-5651-48a5-b822-4a829297b8e1,\n",
       " Recurrence Details: Runs at 0:00 on Monday every Week),\n",
       " Pipeline(Name: Pipeline-Hebdomadaire,\n",
       " Id: 16330cbf-e5eb-4f65-b392-91b358ec2a30,\n",
       " Status: Active,\n",
       " Pipeline Id: 2e74f482-5299-47b4-a4de-e4f346c1a092,\n",
       " Recurrence Details: Runs at 0:00 on Monday every Week),\n",
       " Pipeline(Name: DataDriftSchedule-a699825f,\n",
       " Id: 63a13118-ff67-4104-b786-9aa7d906ff7f,\n",
       " Status: Active,\n",
       " Pipeline Id: 22eb5dff-95b9-4552-bd41-2d12fc358663,\n",
       " Recurrence Details: Runs every Day),\n",
       " Pipeline(Name: Pipeline-Hebdomadaire,\n",
       " Id: aca62762-60cb-4020-bcc0-33869888a755,\n",
       " Status: Active,\n",
       " Pipeline Id: 28a10f90-432f-4903-b1e7-f099d39cac23,\n",
       " Recurrence Details: Runs at 0:00 on Monday every Week),\n",
       " Pipeline(Name: Pipeline-Hebdomadaire,\n",
       " Id: 125ca510-deac-4a5a-b358-7be20cf9d264,\n",
       " Status: Active,\n",
       " Pipeline Id: 9b437e21-ff86-403e-89ec-91b52e4ded1a,\n",
       " Recurrence Details: Runs at 0:00 on Monday every Week),\n",
       " Pipeline(Name: Pipeline-Hebdomadaire,\n",
       " Id: 8b4150bf-3096-49d3-a5d6-1b5aa1c7d1b0,\n",
       " Status: Active,\n",
       " Pipeline Id: 3dcd430c-e1c8-4ae1-bccd-1db3c36beb02,\n",
       " Recurrence Details: Runs at 0:00 on Monday every Week),\n",
       " Pipeline(Name: Pipeline-Hebdomadaire,\n",
       " Id: cffa8f71-4ce3-4101-bada-099cd166b7e8,\n",
       " Status: Active,\n",
       " Pipeline Id: df77077a-97a1-4a51-8ef8-b85df35761fa,\n",
       " Recurrence Details: Runs at 0:00 on Monday every Week),\n",
       " Pipeline(Name: Pipeline-Hebdomadaire,\n",
       " Id: dde1bfc1-5ace-42fc-9aac-83a06dd6a83c,\n",
       " Status: Active,\n",
       " Pipeline Id: f79b994b-9fdc-46e7-92bf-3c305c741c2d,\n",
       " Recurrence Details: Runs at 0:00 on Monday every Week),\n",
       " Pipeline(Name: Pipeline-Hebdomadaire,\n",
       " Id: d36ebb0f-856d-4564-aa87-8f5d00959ea9,\n",
       " Status: Active,\n",
       " Pipeline Id: c0f5f3c4-1386-4492-a757-3ee534042387,\n",
       " Recurrence Details: Runs at 0:00 on Monday every Week),\n",
       " Pipeline(Name: Pipeline-Hebdomadaire,\n",
       " Id: 5bb37fff-417c-40e8-a39e-a11f80cdf8df,\n",
       " Status: Active,\n",
       " Pipeline Id: 673be907-b512-4bb1-b7d1-1107e3c375ea,\n",
       " Recurrence Details: Runs at 0:00 on Monday every Week),\n",
       " Pipeline(Name: Pipeline-Hebdomadaire,\n",
       " Id: a6620f71-e7cd-4307-89da-68431094dabc,\n",
       " Status: Active,\n",
       " Pipeline Id: 9a748ef2-ae8e-4c69-8f92-53707f2b3a8e,\n",
       " Recurrence Details: Runs at 0:00 on Monday every Week),\n",
       " Pipeline(Name: Pipeline-Hebdomadaire,\n",
       " Id: ee6ddd01-69fd-41ba-bfd5-b09c2d2025a7,\n",
       " Status: Active,\n",
       " Pipeline Id: 2a907a5b-eb5d-4b94-bcc5-ecead8d28957,\n",
       " Recurrence Details: Runs at 0:00 on Monday every Week)]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Visualisation des planifications\n",
    "schedules = Schedule.list(ws)\n",
    "schedules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'runId': '6bbe06bd-9b82-42aa-b044-6d56254c9dd0',\n",
       " 'status': 'Completed',\n",
       " 'startTimeUtc': '2020-09-16T13:35:14.587221Z',\n",
       " 'endTimeUtc': '2020-09-16T13:35:28.631687Z',\n",
       " 'properties': {'azureml.runsource': 'azureml.PipelineRun',\n",
       "  'runSource': 'Unavailable',\n",
       "  'runType': 'HTTP',\n",
       "  'azureml.parameters': '{}',\n",
       "  'azureml.pipelineid': '09b07455-b156-474f-9fca-0416aa0de511'},\n",
       " 'inputDatasets': [],\n",
       " 'logFiles': {'logs/azureml/executionlogs.txt': 'https://workshopaml2027584246021.blob.core.windows.net/azureml/ExperimentRun/dcid.6bbe06bd-9b82-42aa-b044-6d56254c9dd0/logs/azureml/executionlogs.txt?sv=2019-02-02&sr=b&sig=iUW79TtEXyAmrA%2Bs1hfUOymcPsaka9bPpXZuSaeyVtY%3D&st=2020-09-16T13%3A25%3A37Z&se=2020-09-16T21%3A35%3A37Z&sp=r',\n",
       "  'logs/azureml/stderrlogs.txt': 'https://workshopaml2027584246021.blob.core.windows.net/azureml/ExperimentRun/dcid.6bbe06bd-9b82-42aa-b044-6d56254c9dd0/logs/azureml/stderrlogs.txt?sv=2019-02-02&sr=b&sig=JUPuEa2wuFsN3OPX2F0NPoum4aK8ysWiQq5kruFIDt0%3D&st=2020-09-16T13%3A25%3A37Z&se=2020-09-16T21%3A35%3A37Z&sp=r',\n",
       "  'logs/azureml/stdoutlogs.txt': 'https://workshopaml2027584246021.blob.core.windows.net/azureml/ExperimentRun/dcid.6bbe06bd-9b82-42aa-b044-6d56254c9dd0/logs/azureml/stdoutlogs.txt?sv=2019-02-02&sr=b&sig=jT7AN%2FpA7IY1cH7GpmbIJSKFhxv2pXpXavAXz6toUiI%3D&st=2020-09-16T13%3A25%3A37Z&se=2020-09-16T21%3A35%3A37Z&sp=r'}}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline_experiment = ws.experiments.get(experiment_name)\n",
    "latest_run = list(pipeline_experiment.get_runs())[0]\n",
    "\n",
    "latest_run.get_details()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Suppression compute server"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Pour supprimer le compute server\n",
    "pipeline_cluster.delete()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cpupipelines  -  AmlCompute : Deleting\n",
      "AutoML  -  AmlCompute : Succeeded\n",
      "Designer  -  AmlCompute : Succeeded\n",
      "notebooksjupyter  -  ComputeInstance : Succeeded\n",
      "cpucluster  -  AmlCompute : Succeeded\n",
      "AutoMLsdk  -  AmlCompute : Succeeded\n",
      "clustergpuNC6  -  AmlCompute : Succeeded\n",
      "gpuclusterstdnc6  -  AmlCompute : Succeeded\n"
     ]
    }
   ],
   "source": [
    "compute_targets = ws.compute_targets\n",
    "for name, ct in compute_targets.items():\n",
    "    print(name, \" - \" , ct.type, \":\", ct.provisioning_state)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> https://docs.microsoft.com/en-us/azure/machine-learning/how-to-use-parallel-run-step"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"https://github.com/retkowsky/images/blob/master/Powered-by-MS-Azure-logo-v2.png?raw=true\" height=\"300\" width=\"300\">"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6 - AzureML",
   "language": "python",
   "name": "python3-azureml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}