3-automl.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Automated ML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import json\n",
    "import pathlib\n",
    "import pprint\n",
    "import requests\n",
    "\n",
    "import joblib\n",
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import azureml.core\n",
    "from azureml.core import Workspace, Experiment\n",
    "from azureml.core.compute import ComputeTarget\n",
    "from azureml.core.compute_target import ComputeTargetException\n",
    "from azureml.core.dataset import Dataset\n",
    "from azureml.widgets import RunDetails"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "gather": {
     "logged": 1598531914256
    }
   },
   "outputs": [],
   "source": [
    "from nd00333.model.automl.train import run_config as automl_train_config\n",
    "from nd00333.model.deploy import run_config as deploy_config\n",
    "from nd00333 import utils as package_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "azureml.core.VERSION 1.21.0\n"
     ]
    }
   ],
   "source": [
    "print(\"azureml.core.VERSION\", azureml.core.VERSION)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create an experiment\n",
    "\n",
    "Create an experiment identified by the creation date. The purpose of identifying the experiments is to not mix the manually run experiments (using this jupyter notebook) with the experiments run using the deployment automation of the master git branch.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "#!az logout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "gather": {
     "logged": 1598531917374
    },
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Failed to load or parse file /home/vscode/.azureml/auth/azureProfile.json. It will be overridden by default settings.\"}\n",
      "{\"message\": \"Failed to load or parse file /home/vscode/.azureml/auth/azureProfile.json. It will be overridden by default settings.\"}\n",
      "{\"message\": \"Failed to load or parse file /home/vscode/.azureml/auth/azureProfile.json. It will be overridden by default settings.\"}\n",
      "{\"message\": \"Failed to load or parse file /home/vscode/.azureml/auth/azureProfile.json. It will be overridden by default settings.\"}\n",
      "{\"message\": \"Failed to load or parse file /home/vscode/.azureml/auth/az.json. It will be overridden by default settings.\"}\n",
      "{\"message\": \"Failed to load or parse file /home/vscode/.azureml/auth/az.json. It will be overridden by default settings.\"}\n",
      "{\"message\": \"Failed to load or parse file /home/vscode/.azureml/auth/az.json. It will be overridden by default settings.\"}\n",
      "{\"message\": \"Failed to load or parse file /home/vscode/.azureml/auth/az.json. It will be overridden by default settings.\"}\n",
      "{\"message\": \"Failed to load or parse file /home/vscode/.azureml/auth/az.sess. It will be overridden by default settings.\"}\n",
      "{\"message\": \"Failed to load or parse file /home/vscode/.azureml/auth/az.sess. It will be overridden by default settings.\"}\n",
      "{\"message\": \"Failed to load or parse file /home/vscode/.azureml/auth/az.sess. It will be overridden by default settings.\"}\n",
      "{\"message\": \"Failed to load or parse file /home/vscode/.azureml/auth/az.sess. It will be overridden by default settings.\"}\n",
      "{\"message\": \"Failed to load or parse file /home/vscode/.azure/azureProfile.json. It will be overridden by default settings.\"}\n",
      "{\"message\": \"Failed to load or parse file /home/vscode/.azure/azureProfile.json. It will be overridden by default settings.\"}\n",
      "{\"message\": \"Failed to load or parse file /home/vscode/.azure/azureProfile.json. It will be overridden by default settings.\"}\n",
      "{\"message\": \"Failed to load or parse file /home/vscode/.azure/azureProfile.json. It will be overridden by default settings.\"}\n",
      "{\"message\": \"Detect no GUI is available, so fall back to device code\"}\n",
      "{\"message\": \"Detect no GUI is available, so fall back to device code\"}\n",
      "{\"message\": \"Detect no GUI is available, so fall back to device code\"}\n",
      "{\"message\": \"Detect no GUI is available, so fall back to device code\"}\n",
      "{\"message\": \"55b7f2d4-c8e3-430a-973c-c62c58b35fc0 - CodeRequest:Getting user code info.\"}\n",
      "{\"message\": \"55b7f2d4-c8e3-430a-973c-c62c58b35fc0 - CodeRequest:Getting user code info.\"}\n",
      "{\"message\": \"55b7f2d4-c8e3-430a-973c-c62c58b35fc0 - CodeRequest:Getting user code info.\"}\n",
      "{\"message\": \"55b7f2d4-c8e3-430a-973c-c62c58b35fc0 - CodeRequest:Getting user code info.\"}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performing interactive authentication. Please follow the instructions on the terminal.\n",
      "To sign in, use a web browser to open the page https://microsoft.com/devicelogin and enter the code DWREUYXJ2 to authenticate.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"message\": \"55b7f2d4-c8e3-430a-973c-c62c58b35fc0 - TokenRequest:Getting a token via device code\"}\n",
      "{\"message\": \"55b7f2d4-c8e3-430a-973c-c62c58b35fc0 - TokenRequest:Getting a token via device code\"}\n",
      "{\"message\": \"55b7f2d4-c8e3-430a-973c-c62c58b35fc0 - TokenRequest:Getting a token via device code\"}\n",
      "{\"message\": \"55b7f2d4-c8e3-430a-973c-c62c58b35fc0 - TokenRequest:Getting a token via device code\"}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You have logged in. Now let us find all the subscriptions to which you have access...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"message\": \"b991fd3f-0ccb-44f7-b848-447764223503 - CacheDriver:Acquiring new access token from MRRT token.\"}\n",
      "{\"message\": \"b991fd3f-0ccb-44f7-b848-447764223503 - CacheDriver:Acquiring new access token from MRRT token.\"}\n",
      "{\"message\": \"b991fd3f-0ccb-44f7-b848-447764223503 - CacheDriver:Acquiring new access token from MRRT token.\"}\n",
      "{\"message\": \"b991fd3f-0ccb-44f7-b848-447764223503 - CacheDriver:Acquiring new access token from MRRT token.\"}\n",
      "{\"message\": \"b991fd3f-0ccb-44f7-b848-447764223503 - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"b991fd3f-0ccb-44f7-b848-447764223503 - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"b991fd3f-0ccb-44f7-b848-447764223503 - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"b991fd3f-0ccb-44f7-b848-447764223503 - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"b991fd3f-0ccb-44f7-b848-447764223503 - CacheDriver:Returning token derived from mrrt refresh.\"}\n",
      "{\"message\": \"b991fd3f-0ccb-44f7-b848-447764223503 - CacheDriver:Returning token derived from mrrt refresh.\"}\n",
      "{\"message\": \"b991fd3f-0ccb-44f7-b848-447764223503 - CacheDriver:Returning token derived from mrrt refresh.\"}\n",
      "{\"message\": \"b991fd3f-0ccb-44f7-b848-447764223503 - CacheDriver:Returning token derived from mrrt refresh.\"}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Interactive authentication successfully completed.\n"
     ]
    }
   ],
   "source": [
    "workspace = Workspace.from_config()\n",
    "utcnow = datetime.datetime.utcnow().strftime(\"%Y-%m-%d-%H-%m\")\n",
    "utcnow = \"2021-01-29-09-02\"\n",
    "experiment_name = f\"jupyter-{utcnow}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"message\": \"Created a worker pool for first use\"}\n",
      "{\"message\": \"Created a worker pool for first use\"}\n",
      "{\"message\": \"Created a worker pool for first use\"}\n",
      "{\"message\": \"Created a worker pool for first use\"}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "experiment_name jupyter-2021-01-29-09-02\n"
     ]
    }
   ],
   "source": [
    "experiment = Experiment(workspace, experiment_name)\n",
    "print(f\"experiment_name {experiment_name}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset\n",
    "\n",
    "Load the dataset from local files on disk into the default AzureML workspace datastore.\n",
    "Store the versioned (by the \"_1\" suffix), combined train+validate dataset for the purpose of cross-validation, and test sets under separate datastore paths. The uploading of large datasets tends to break with \"The write operation timed out\" when using the Python SDK and therefore the `az` is used. See https://docs.microsoft.com/en-us/answers/questions/43980/cannot-upload-local-files-to-azureml-datastore-pyt.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_trainandvalidate_name = \"ids2018trainandvalidate_1\"\n",
    "dataset_test_name = \"ids2018test_1\"\n",
    "dataset_2017_name = \"ids2017full_1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mTo sign in, use a web browser to open the page https://microsoft.com/devicelogin and enter the code D9ATJRDZL to authenticate.\u001b[0m\n",
      "[\n",
      "  {\n",
      "    \"cloudName\": \"AzureCloud\",\n",
      "    \"homeTenantId\": \"660b3398-b80e-49d2-bc5b-ac1dc93b5254\",\n",
      "    \"id\": \"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\",\n",
      "    \"isDefault\": true,\n",
      "    \"managedByTenants\": [],\n",
      "    \"name\": \"Udacity CloudLabs Sub - 09\",\n",
      "    \"state\": \"Enabled\",\n",
      "    \"tenantId\": \"660b3398-b80e-49d2-bc5b-ac1dc93b5254\",\n",
      "    \"user\": {\n",
      "      \"name\": \"odl_user_136633@udacitylabs.onmicrosoft.com\",\n",
      "      \"type\": \"user\"\n",
      "    }\n",
      "  }\n",
      "]\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!az login"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0m- Installing ..."
     ]
    }
   ],
   "source": [
    "!az extension add --name azure-cli-ml --version 1.21.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "azure-cli                         2.18.0\n",
      "\n",
      "core                              2.18.0\n",
      "telemetry                          1.0.6\n",
      "\n",
      "Extensions:\n",
      "azure-cli-ml                      1.21.0\n",
      "\n",
      "Python location '/opt/az/bin/python3'\n",
      "Extensions directory '/home/vscode/.azure/cliextensions'\n",
      "\n",
      "Python (Linux) 3.6.10 (default, Jan 15 2021, 09:54:35) \n",
      "[GCC 8.3.0]\n",
      "\n",
      "Legal docs and information: aka.ms/AzureCliLegal\n",
      "\n",
      "\n",
      "Your CLI is up-to-date.\n",
      "\n",
      "\u001b[33m\u001b[1mPlease let us know how we are doing: \u001b[34mhttps://aka.ms/azureclihats\u001b[0m\n",
      "\u001b[33m\u001b[1mand let us know if you're interested in trying out our newest features: \u001b[34mhttps://aka.ms/CLIUXstudy\u001b[0m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!az --version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mFound the config file in: /app/config.json\u001b[0m\n",
      "Uploading an estimated of 1 files\n",
      "Uploading datasets/ids2018trainandvalidate/data.csv\n",
      "Uploaded datasets/ids2018trainandvalidate/data.csv, 1 files out of an estimated total of 1\n",
      "Uploaded 1 files\n",
      "\u001b[32mCommand ran in 900.310 seconds (init: 0.385, invoke: 899.925)\u001b[0m\n",
      "\u001b[0mCPU times: user 54.3 s, sys: 10.5 s, total: 1min 4s\n",
      "Wall time: 15min 1s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "!az ml datastore upload --name workspaceblobstore --verbose \\\n",
    "    --src-path datasets/ids2018trainandvalidate --target-path \\\n",
    "    $dataset_trainandvalidate_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mFound the config file in: /app/config.json\u001b[0m\n",
      "Uploading an estimated of 1 files\n",
      "Uploading datasets/ids2018test/data.csv\n",
      "Uploaded datasets/ids2018test/data.csv, 1 files out of an estimated total of 1\n",
      "Uploaded 1 files\n",
      "\u001b[32mCommand ran in 527.825 seconds (init: 0.411, invoke: 527.414)\u001b[0m\n",
      "\u001b[0mCPU times: user 30.3 s, sys: 5.69 s, total: 36 s\n",
      "Wall time: 8min 49s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "!az ml datastore upload --name workspaceblobstore --verbose \\\n",
    "    --src-path datasets/ids2018test --target-path \\\n",
    "    $dataset_test_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32mFound the config file in: /app/config.json\u001b[0m\n",
      "Uploading an estimated of 8 files\n",
      "Uploading datasets/ids2017full/Friday-WorkingHours-Afternoon-PortScan.pcap_ISCX.csv\n",
      "Uploaded datasets/ids2017full/Friday-WorkingHours-Afternoon-PortScan.pcap_ISCX.csv, 1 files out of an estimated total of 8\n",
      "Uploading datasets/ids2017full/Friday-WorkingHours-Afternoon-DDos.pcap_ISCX.csv\n",
      "Uploaded datasets/ids2017full/Friday-WorkingHours-Afternoon-DDos.pcap_ISCX.csv, 2 files out of an estimated total of 8\n",
      "Uploading datasets/ids2017full/Friday-WorkingHours-Morning.pcap_ISCX.csv\n",
      "Uploaded datasets/ids2017full/Friday-WorkingHours-Morning.pcap_ISCX.csv, 3 files out of an estimated total of 8\n",
      "Uploading datasets/ids2017full/Thursday-WorkingHours-Morning-WebAttacks.pcap_ISCX.csv\n",
      "Uploaded datasets/ids2017full/Thursday-WorkingHours-Morning-WebAttacks.pcap_ISCX.csv, 4 files out of an estimated total of 8\n",
      "Uploading datasets/ids2017full/Thursday-WorkingHours-Afternoon-Infilteration.pcap_ISCX.csv\n",
      "Uploaded datasets/ids2017full/Thursday-WorkingHours-Afternoon-Infilteration.pcap_ISCX.csv, 5 files out of an estimated total of 8\n",
      "Uploading datasets/ids2017full/Tuesday-WorkingHours.pcap_ISCX.csv\n",
      "Uploaded datasets/ids2017full/Tuesday-WorkingHours.pcap_ISCX.csv, 6 files out of an estimated total of 8\n",
      "Uploading datasets/ids2017full/Monday-WorkingHours.pcap_ISCX.csv\n",
      "Uploaded datasets/ids2017full/Monday-WorkingHours.pcap_ISCX.csv, 7 files out of an estimated total of 8\n",
      "Uploading datasets/ids2017full/Wednesday-workingHours.pcap_ISCX.csv\n",
      "Uploaded datasets/ids2017full/Wednesday-workingHours.pcap_ISCX.csv, 8 files out of an estimated total of 8\n",
      "Uploaded 8 files\n",
      "\u001b[32mCommand ran in 208.126 seconds (init: 0.267, invoke: 207.859)\u001b[0m\n",
      "\u001b[0mCPU times: user 11.3 s, sys: 2.14 s, total: 13.5 s\n",
      "Wall time: 3min 29s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "!az ml datastore upload --name workspaceblobstore --verbose \\\n",
    "    --src-path datasets/ids2017full --target-path \\\n",
    "    $dataset_2017_name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Upload and register the uploaded datasets into AzureML workspace. In principle the above `az` commands should not be necessary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"<azureml.core.authentication.InteractiveLoginAuthentication object at 0x7f52db02cfd0>\"}\n",
      "{\"message\": \"datastore.upload_files\", \"kwargs\": {\"files\": [\"/app/datasets/ids2018trainandvalidate/data.csv\"], \"target_path\": \"ids2018trainandvalidate_1\", \"overwrite\": false}}\n",
      "{\"message\": \"Called AzureBlobDatastore.upload_files\"}\n",
      "Uploading an estimated of 1 files\n",
      "{\"message\": \"Uploading an estimated of 1 files\"}\n",
      "Target already exists. Skipping upload for ids2018trainandvalidate_1/data.csv\n",
      "{\"message\": \"Target already exists. Skipping upload for ids2018trainandvalidate_1/data.csv\"}\n",
      "Uploaded 0 files\n",
      "{\"message\": \"Uploaded 0 files\"}\n",
      "{\"message\": \"Finished AzureBlobDatastore.upload with count=0.\"}\n",
      "{\"message\": \"TabularDatasetFactory.from_delimited_files\", \"kwargs\": {\"path\": [\"<azureml.data.datapath.DataPath object at 0x7f52dafe8b50>\"], \"set_column_types\": {\"Flow Duration\": \"<azureml.data.dataset_factory.DataType object at 0x7f52db35a710>\", \"TotLen Fwd Pkts\": \"<azureml.data.dataset_factory.DataType object at 0x7f52db35a710>\", \"TotLen Bwd Pkts\": \"<azureml.data.dataset_factory.DataType object at 0x7f52db35a710>\", \"Fwd Pkt Len Std\": \"<azureml.data.dataset_factory.DataType object at 0x7f52db35a710>\", \"Bwd Pkt Len Max\": \"<azureml.data.dataset_factory.DataType object at 0x7f52db35a710>\", \"Bwd Pkt Len Std\": \"<azureml.data.dataset_factory.DataType object at 0x7f52db35a710>\", \"Flow Byts/s\": \"<azureml.data.dataset_factory.DataType object at 0x7f52db35a710>\", \"Flow Pkts/s\": \"<azureml.data.dataset_factory.DataType object at 0x7f52db35a710>\", \"Flow IAT Max\": \"<azureml.data.dataset_factory.DataType object at 0x7f52db35a710>\", \"Bwd IAT Min\": \"<azureml.data.dataset_factory.DataType object at 0x7f52db35a710>\", \"Bwd Header Len\": \"<azureml.data.dataset_factory.DataType object at 0x7f52db35a710>\", \"Pkt Len Max\": \"<azureml.data.dataset_factory.DataType object at 0x7f52db35a710>\", \"Pkt Len Std\": \"<azureml.data.dataset_factory.DataType object at 0x7f52db35a710>\", \"RST Flag Cnt\": \"<azureml.data.dataset_factory.DataType object at 0x7f52db35a710>\", \"PSH Flag Cnt\": \"<azureml.data.dataset_factory.DataType object at 0x7f52db35a710>\", \"ECE Flag Cnt\": \"<azureml.data.dataset_factory.DataType object at 0x7f52db35a710>\", \"Init Fwd Win Byts\": \"<azureml.data.dataset_factory.DataType object at 0x7f52db35a710>\", \"Init Bwd Win Byts\": \"<azureml.data.dataset_factory.DataType object at 0x7f52db35a710>\", \"Fwd Seg Size Min\": \"<azureml.data.dataset_factory.DataType object at 0x7f52db35a710>\", \"Label\": \"<azureml.data.dataset_factory.DataType object at 0x7f52db35d510>\"}}}\n",
      "{\"message\": \"tabular.register\", \"kwargs\": {\"workspace\": \"Workspace.create(name='quick-starts-ws-136633', subscription_id='a24a24d5-8d87-4c8a-99b6-91ed2d2df51f', resource_group='aml-quickstarts-136633')\", \"name\": \"ids2018trainandvalidate_1\", \"create_new_version\": false}}\n",
      "CPU times: user 11.8 s, sys: 2.14 s, total: 13.9 s\n",
      "Wall time: 3min 27s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "!PYTHONPATH=. python nd00333/dataset/register/register.py \\\n",
    "    --dataset-path datasets --dataset-name ids2018trainandvalidate --dataset-version 1 \\\n",
    "    --dataset-type tabular"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"<azureml.core.authentication.InteractiveLoginAuthentication object at 0x7f065e4ec250>\"}\n",
      "{\"message\": \"datastore.upload_files\", \"kwargs\": {\"files\": [\"/app/datasets/ids2018test/data.csv\"], \"target_path\": \"ids2018test_1\", \"overwrite\": false}}\n",
      "{\"message\": \"Called AzureBlobDatastore.upload_files\"}\n",
      "Uploading an estimated of 1 files\n",
      "{\"message\": \"Uploading an estimated of 1 files\"}\n",
      "Target already exists. Skipping upload for ids2018test_1/data.csv\n",
      "{\"message\": \"Target already exists. Skipping upload for ids2018test_1/data.csv\"}\n",
      "Uploaded 0 files\n",
      "{\"message\": \"Uploaded 0 files\"}\n",
      "{\"message\": \"Finished AzureBlobDatastore.upload with count=0.\"}\n",
      "{\"message\": \"TabularDatasetFactory.from_delimited_files\", \"kwargs\": {\"path\": [\"<azureml.data.datapath.DataPath object at 0x7f065e506150>\"], \"set_column_types\": {\"Flow Duration\": \"<azureml.data.dataset_factory.DataType object at 0x7f066504d410>\", \"TotLen Fwd Pkts\": \"<azureml.data.dataset_factory.DataType object at 0x7f066504d410>\", \"TotLen Bwd Pkts\": \"<azureml.data.dataset_factory.DataType object at 0x7f066504d410>\", \"Fwd Pkt Len Std\": \"<azureml.data.dataset_factory.DataType object at 0x7f066504d410>\", \"Bwd Pkt Len Max\": \"<azureml.data.dataset_factory.DataType object at 0x7f066504d410>\", \"Bwd Pkt Len Std\": \"<azureml.data.dataset_factory.DataType object at 0x7f066504d410>\", \"Flow Byts/s\": \"<azureml.data.dataset_factory.DataType object at 0x7f066504d410>\", \"Flow Pkts/s\": \"<azureml.data.dataset_factory.DataType object at 0x7f066504d410>\", \"Flow IAT Max\": \"<azureml.data.dataset_factory.DataType object at 0x7f066504d410>\", \"Bwd IAT Min\": \"<azureml.data.dataset_factory.DataType object at 0x7f066504d410>\", \"Bwd Header Len\": \"<azureml.data.dataset_factory.DataType object at 0x7f066504d410>\", \"Pkt Len Max\": \"<azureml.data.dataset_factory.DataType object at 0x7f066504d410>\", \"Pkt Len Std\": \"<azureml.data.dataset_factory.DataType object at 0x7f066504d410>\", \"RST Flag Cnt\": \"<azureml.data.dataset_factory.DataType object at 0x7f066504d410>\", \"PSH Flag Cnt\": \"<azureml.data.dataset_factory.DataType object at 0x7f066504d410>\", \"ECE Flag Cnt\": \"<azureml.data.dataset_factory.DataType object at 0x7f066504d410>\", \"Init Fwd Win Byts\": \"<azureml.data.dataset_factory.DataType object at 0x7f066504d410>\", \"Init Bwd Win Byts\": \"<azureml.data.dataset_factory.DataType object at 0x7f066504d410>\", \"Fwd Seg Size Min\": \"<azureml.data.dataset_factory.DataType object at 0x7f066504d410>\", \"Label\": \"<azureml.data.dataset_factory.DataType object at 0x7f065e883490>\"}}}\n",
      "{\"message\": \"tabular.register\", \"kwargs\": {\"workspace\": \"Workspace.create(name='quick-starts-ws-136633', subscription_id='a24a24d5-8d87-4c8a-99b6-91ed2d2df51f', resource_group='aml-quickstarts-136633')\", \"name\": \"ids2018test_1\", \"create_new_version\": false}}\n",
      "CPU times: user 3.97 s, sys: 840 ms, total: 4.81 s\n",
      "Wall time: 1min 10s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "!PYTHONPATH=. python nd00333/dataset/register/register.py \\\n",
    "     --dataset-path datasets --dataset-name ids2018test --dataset-version 1 \\\n",
    "     --dataset-type tabular"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"<azureml.core.authentication.InteractiveLoginAuthentication object at 0x7f81d1ba5390>\"}\n",
      "{\"message\": \"datastore.upload_files\", \"kwargs\": {\"files\": [\"/app/datasets/ids2017full/Friday-WorkingHours-Afternoon-DDos.pcap_ISCX.csv\", \"/app/datasets/ids2017full/Friday-WorkingHours-Afternoon-PortScan.pcap_ISCX.csv\", \"/app/datasets/ids2017full/Friday-WorkingHours-Morning.pcap_ISCX.csv\", \"/app/datasets/ids2017full/Monday-WorkingHours.pcap_ISCX.csv\", \"/app/datasets/ids2017full/Thursday-WorkingHours-Afternoon-Infilteration.pcap_ISCX.csv\", \"/app/datasets/ids2017full/Thursday-WorkingHours-Morning-WebAttacks.pcap_ISCX.csv\", \"/app/datasets/ids2017full/Tuesday-WorkingHours.pcap_ISCX.csv\", \"/app/datasets/ids2017full/Wednesday-workingHours.pcap_ISCX.csv\"], \"target_path\": \"ids2017full_1\", \"overwrite\": false}}\n",
      "{\"message\": \"Called AzureBlobDatastore.upload_files\"}\n",
      "Uploading an estimated of 8 files\n",
      "{\"message\": \"Uploading an estimated of 8 files\"}\n",
      "Target already exists. Skipping upload for ids2017full_1/Friday-WorkingHours-Afternoon-DDos.pcap_ISCX.csv\n",
      "{\"message\": \"Target already exists. Skipping upload for ids2017full_1/Friday-WorkingHours-Afternoon-DDos.pcap_ISCX.csv\"}\n",
      "Target already exists. Skipping upload for ids2017full_1/Friday-WorkingHours-Afternoon-PortScan.pcap_ISCX.csv\n",
      "{\"message\": \"Target already exists. Skipping upload for ids2017full_1/Friday-WorkingHours-Afternoon-PortScan.pcap_ISCX.csv\"}\n",
      "Target already exists. Skipping upload for ids2017full_1/Friday-WorkingHours-Morning.pcap_ISCX.csv\n",
      "{\"message\": \"Target already exists. Skipping upload for ids2017full_1/Friday-WorkingHours-Morning.pcap_ISCX.csv\"}\n",
      "Target already exists. Skipping upload for ids2017full_1/Monday-WorkingHours.pcap_ISCX.csv\n",
      "{\"message\": \"Target already exists. Skipping upload for ids2017full_1/Monday-WorkingHours.pcap_ISCX.csv\"}\n",
      "Target already exists. Skipping upload for ids2017full_1/Thursday-WorkingHours-Afternoon-Infilteration.pcap_ISCX.csv\n",
      "{\"message\": \"Target already exists. Skipping upload for ids2017full_1/Thursday-WorkingHours-Afternoon-Infilteration.pcap_ISCX.csv\"}\n",
      "Target already exists. Skipping upload for ids2017full_1/Thursday-WorkingHours-Morning-WebAttacks.pcap_ISCX.csv\n",
      "{\"message\": \"Target already exists. Skipping upload for ids2017full_1/Thursday-WorkingHours-Morning-WebAttacks.pcap_ISCX.csv\"}\n",
      "Target already exists. Skipping upload for ids2017full_1/Tuesday-WorkingHours.pcap_ISCX.csv\n",
      "{\"message\": \"Target already exists. Skipping upload for ids2017full_1/Tuesday-WorkingHours.pcap_ISCX.csv\"}\n",
      "Target already exists. Skipping upload for ids2017full_1/Wednesday-workingHours.pcap_ISCX.csv\n",
      "{\"message\": \"Target already exists. Skipping upload for ids2017full_1/Wednesday-workingHours.pcap_ISCX.csv\"}\n",
      "Uploaded 0 files\n",
      "{\"message\": \"Uploaded 0 files\"}\n",
      "{\"message\": \"Finished AzureBlobDatastore.upload with count=0.\"}\n",
      "{\"message\": \"TabularDatasetFactory.from_delimited_files\", \"kwargs\": {\"path\": [\"<azureml.data.datapath.DataPath object at 0x7f81d1b8d9d0>\", \"<azureml.data.datapath.DataPath object at 0x7f81d1b8d990>\", \"<azureml.data.datapath.DataPath object at 0x7f81d1b8db10>\", \"<azureml.data.datapath.DataPath object at 0x7f81d1b3f250>\", \"<azureml.data.datapath.DataPath object at 0x7f81d1b3f2d0>\", \"<azureml.data.datapath.DataPath object at 0x7f81d1b3f390>\", \"<azureml.data.datapath.DataPath object at 0x7f81d1b3fb90>\", \"<azureml.data.datapath.DataPath object at 0x7f81d1b3f590>\"], \"set_column_types\": {\"Flow Duration\": \"<azureml.data.dataset_factory.DataType object at 0x7f81d1ed5850>\", \"TotLen Fwd Pkts\": \"<azureml.data.dataset_factory.DataType object at 0x7f81d1ed5850>\", \"TotLen Bwd Pkts\": \"<azureml.data.dataset_factory.DataType object at 0x7f81d1ed5850>\", \"Fwd Pkt Len Std\": \"<azureml.data.dataset_factory.DataType object at 0x7f81d1ed5850>\", \"Bwd Pkt Len Max\": \"<azureml.data.dataset_factory.DataType object at 0x7f81d1ed5850>\", \"Bwd Pkt Len Std\": \"<azureml.data.dataset_factory.DataType object at 0x7f81d1ed5850>\", \"Flow Byts/s\": \"<azureml.data.dataset_factory.DataType object at 0x7f81d1ed5850>\", \"Flow Pkts/s\": \"<azureml.data.dataset_factory.DataType object at 0x7f81d1ed5850>\", \"Flow IAT Max\": \"<azureml.data.dataset_factory.DataType object at 0x7f81d1ed5850>\", \"Bwd IAT Min\": \"<azureml.data.dataset_factory.DataType object at 0x7f81d1ed5850>\", \"Bwd Header Len\": \"<azureml.data.dataset_factory.DataType object at 0x7f81d1ed5850>\", \"Pkt Len Max\": \"<azureml.data.dataset_factory.DataType object at 0x7f81d1ed5850>\", \"Pkt Len Std\": \"<azureml.data.dataset_factory.DataType object at 0x7f81d1ed5850>\", \"RST Flag Cnt\": \"<azureml.data.dataset_factory.DataType object at 0x7f81d1ed5850>\", \"PSH Flag Cnt\": \"<azureml.data.dataset_factory.DataType object at 0x7f81d1ed5850>\", \"ECE Flag Cnt\": \"<azureml.data.dataset_factory.DataType object at 0x7f81d1ed5850>\", \"Init Fwd Win Byts\": \"<azureml.data.dataset_factory.DataType object at 0x7f81d1ed5850>\", \"Init Bwd Win Byts\": \"<azureml.data.dataset_factory.DataType object at 0x7f81d1ed5850>\", \"Fwd Seg Size Min\": \"<azureml.data.dataset_factory.DataType object at 0x7f81d1ed5850>\", \"Label\": \"<azureml.data.dataset_factory.DataType object at 0x7f81d1ed7650>\"}}}\n",
      "{\"message\": \"tabular.register\", \"kwargs\": {\"workspace\": \"Workspace.create(name='quick-starts-ws-136633', subscription_id='a24a24d5-8d87-4c8a-99b6-91ed2d2df51f', resource_group='aml-quickstarts-136633')\", \"name\": \"ids2017full_1\", \"create_new_version\": false}}\n",
      "CPU times: user 2.11 s, sys: 433 ms, total: 2.54 s\n",
      "Wall time: 51.1 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "!PYTHONPATH=. python nd00333/dataset/register/register.py \\\n",
    "     --dataset-path datasets --dataset-name ids2017full --dataset-version 1 \\\n",
    "     --dataset-type tabular"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "gather": {
     "logged": 1598531923519
    },
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    }
   },
   "source": [
    "## AutoML Configuration\n",
    "\n",
    "It should be noted that several settings that may affect the model performance are used in AutoMLConfig in order to speed up the training:\n",
    "- the number of `iterations` (individual model runs) is limited to 15,\n",
    "- instead of a cross-validation method, a single split into the training and validation data is specified using `validation_size`=0.3 in order to reduce the training time,\n",
    "- the slow `enable_stack_ensemble` ensemble model is excluded,\n",
    "- only the \"LightGBM\", \"LogisticRegression\", \"SGD\", \"XGBoostClassifier\" models are allowed in the runs in `allowed_models`. Models from the \"RandomForest\" family unbounded by `max_depth` may grow very large (to e.g. several hundreds Mbytes).\n",
    "\n",
    "The individual model runs are performed in parallel using all cores available on the compute instance by `max_cores_per_iteration=`-1. The number of `max_concurrent_iterations` is set to the number of the nodes in the compute cluster.\n",
    "\n",
    "See [nd00333/model/automl/train/run_config.py](nd00333/model/automl/train/run_config.py) for more details.\n",
    "\n",
    "The `norm_macro_recall` is used as the performance metrics. See [README.md](README.md) for the rationale."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Perform the configuration of the AutoML run, including setting up the remote AML compute cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "gather": {
     "logged": 1598544893076
    },
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating\n",
      "Succeeded\n",
      "AmlCompute wait for completion finished\n",
      "\n",
      "Minimum number of nodes requested have been provisioned\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"message\": \"main\", \"status\": {\"currentNodeCount\": 0, \"targetNodeCount\": 0, \"nodeStateCounts\": {\"preparingNodeCount\": 0, \"runningNodeCount\": 0, \"idleNodeCount\": 0, \"unusableNodeCount\": 0, \"leavingNodeCount\": 0, \"preemptedNodeCount\": 0}, \"allocationState\": \"Steady\", \"allocationStateTransitionTime\": \"2021-01-30T19:03:01.056000+00:00\", \"errors\": null, \"creationTime\": \"2021-01-30T19:02:55.373586+00:00\", \"modifiedTime\": \"2021-01-30T19:03:10.789141+00:00\", \"provisioningState\": \"Succeeded\", \"provisioningStateTransitionTime\": null, \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"vmPriority\": \"LowPriority\", \"vmSize\": \"STANDARD_D12_V2\"}}\n",
      "{\"message\": \"main\", \"status\": {\"currentNodeCount\": 0, \"targetNodeCount\": 0, \"nodeStateCounts\": {\"preparingNodeCount\": 0, \"runningNodeCount\": 0, \"idleNodeCount\": 0, \"unusableNodeCount\": 0, \"leavingNodeCount\": 0, \"preemptedNodeCount\": 0}, \"allocationState\": \"Steady\", \"allocationStateTransitionTime\": \"2021-01-30T19:03:01.056000+00:00\", \"errors\": null, \"creationTime\": \"2021-01-30T19:02:55.373586+00:00\", \"modifiedTime\": \"2021-01-30T19:03:10.789141+00:00\", \"provisioningState\": \"Succeeded\", \"provisioningStateTransitionTime\": null, \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"vmPriority\": \"LowPriority\", \"vmSize\": \"STANDARD_D12_V2\"}}\n",
      "{\"message\": \"main\", \"status\": {\"currentNodeCount\": 0, \"targetNodeCount\": 0, \"nodeStateCounts\": {\"preparingNodeCount\": 0, \"runningNodeCount\": 0, \"idleNodeCount\": 0, \"unusableNodeCount\": 0, \"leavingNodeCount\": 0, \"preemptedNodeCount\": 0}, \"allocationState\": \"Steady\", \"allocationStateTransitionTime\": \"2021-01-30T19:03:01.056000+00:00\", \"errors\": null, \"creationTime\": \"2021-01-30T19:02:55.373586+00:00\", \"modifiedTime\": \"2021-01-30T19:03:10.789141+00:00\", \"provisioningState\": \"Succeeded\", \"provisioningStateTransitionTime\": null, \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"vmPriority\": \"LowPriority\", \"vmSize\": \"STANDARD_D12_V2\"}}\n",
      "{\"message\": \"main\", \"status\": {\"currentNodeCount\": 0, \"targetNodeCount\": 0, \"nodeStateCounts\": {\"preparingNodeCount\": 0, \"runningNodeCount\": 0, \"idleNodeCount\": 0, \"unusableNodeCount\": 0, \"leavingNodeCount\": 0, \"preemptedNodeCount\": 0}, \"allocationState\": \"Steady\", \"allocationStateTransitionTime\": \"2021-01-30T19:03:01.056000+00:00\", \"errors\": null, \"creationTime\": \"2021-01-30T19:02:55.373586+00:00\", \"modifiedTime\": \"2021-01-30T19:03:10.789141+00:00\", \"provisioningState\": \"Succeeded\", \"provisioningStateTransitionTime\": null, \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"vmPriority\": \"LowPriority\", \"vmSize\": \"STANDARD_D12_V2\"}}\n",
      "{\"message\": \"main\", \"compute_target\": {\"id\": \"/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/computes/starts-ws-136633\", \"name\": \"starts-ws-136633\", \"location\": \"southcentralus\", \"tags\": null, \"identity\": null, \"properties\": {\"description\": null, \"resourceId\": null, \"computeType\": \"AmlCompute\", \"computeLocation\": \"southcentralus\", \"provisioningState\": \"Succeeded\", \"provisioningErrors\": null, \"properties\": {\"vmSize\": \"STANDARD_D12_V2\", \"vmPriority\": \"LowPriority\", \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"userAccountCredentials\": {\"adminUserName\": null, \"adminUserPassword\": null, \"adminUserSshPublicKey\": null}, \"subnet\": {\"id\": null}, \"remoteLoginPortPublicAccess\": \"Enabled\"}, \"status\": {\"currentNodeCount\": 0, \"targetNodeCount\": 0, \"nodeStateCounts\": {\"preparingNodeCount\": 0, \"runningNodeCount\": 0, \"idleNodeCount\": 0, \"unusableNodeCount\": 0, \"leavingNodeCount\": 0, \"preemptedNodeCount\": 0}, \"allocationState\": \"Steady\", \"allocationStateTransitionTime\": \"2021-01-30T19:03:01.056000+00:00\", \"errors\": null, \"creationTime\": \"2021-01-30T19:02:55.373586+00:00\", \"modifiedTime\": \"2021-01-30T19:03:10.789141+00:00\", \"provisioningState\": \"Succeeded\", \"provisioningStateTransitionTime\": null, \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"vmPriority\": \"LowPriority\", \"vmSize\": \"STANDARD_D12_V2\"}}}}\n",
      "{\"message\": \"main\", \"compute_target\": {\"id\": \"/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/computes/starts-ws-136633\", \"name\": \"starts-ws-136633\", \"location\": \"southcentralus\", \"tags\": null, \"identity\": null, \"properties\": {\"description\": null, \"resourceId\": null, \"computeType\": \"AmlCompute\", \"computeLocation\": \"southcentralus\", \"provisioningState\": \"Succeeded\", \"provisioningErrors\": null, \"properties\": {\"vmSize\": \"STANDARD_D12_V2\", \"vmPriority\": \"LowPriority\", \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"userAccountCredentials\": {\"adminUserName\": null, \"adminUserPassword\": null, \"adminUserSshPublicKey\": null}, \"subnet\": {\"id\": null}, \"remoteLoginPortPublicAccess\": \"Enabled\"}, \"status\": {\"currentNodeCount\": 0, \"targetNodeCount\": 0, \"nodeStateCounts\": {\"preparingNodeCount\": 0, \"runningNodeCount\": 0, \"idleNodeCount\": 0, \"unusableNodeCount\": 0, \"leavingNodeCount\": 0, \"preemptedNodeCount\": 0}, \"allocationState\": \"Steady\", \"allocationStateTransitionTime\": \"2021-01-30T19:03:01.056000+00:00\", \"errors\": null, \"creationTime\": \"2021-01-30T19:02:55.373586+00:00\", \"modifiedTime\": \"2021-01-30T19:03:10.789141+00:00\", \"provisioningState\": \"Succeeded\", \"provisioningStateTransitionTime\": null, \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"vmPriority\": \"LowPriority\", \"vmSize\": \"STANDARD_D12_V2\"}}}}\n",
      "{\"message\": \"main\", \"compute_target\": {\"id\": \"/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/computes/starts-ws-136633\", \"name\": \"starts-ws-136633\", \"location\": \"southcentralus\", \"tags\": null, \"identity\": null, \"properties\": {\"description\": null, \"resourceId\": null, \"computeType\": \"AmlCompute\", \"computeLocation\": \"southcentralus\", \"provisioningState\": \"Succeeded\", \"provisioningErrors\": null, \"properties\": {\"vmSize\": \"STANDARD_D12_V2\", \"vmPriority\": \"LowPriority\", \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"userAccountCredentials\": {\"adminUserName\": null, \"adminUserPassword\": null, \"adminUserSshPublicKey\": null}, \"subnet\": {\"id\": null}, \"remoteLoginPortPublicAccess\": \"Enabled\"}, \"status\": {\"currentNodeCount\": 0, \"targetNodeCount\": 0, \"nodeStateCounts\": {\"preparingNodeCount\": 0, \"runningNodeCount\": 0, \"idleNodeCount\": 0, \"unusableNodeCount\": 0, \"leavingNodeCount\": 0, \"preemptedNodeCount\": 0}, \"allocationState\": \"Steady\", \"allocationStateTransitionTime\": \"2021-01-30T19:03:01.056000+00:00\", \"errors\": null, \"creationTime\": \"2021-01-30T19:02:55.373586+00:00\", \"modifiedTime\": \"2021-01-30T19:03:10.789141+00:00\", \"provisioningState\": \"Succeeded\", \"provisioningStateTransitionTime\": null, \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"vmPriority\": \"LowPriority\", \"vmSize\": \"STANDARD_D12_V2\"}}}}\n",
      "{\"message\": \"main\", \"compute_target\": {\"id\": \"/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/computes/starts-ws-136633\", \"name\": \"starts-ws-136633\", \"location\": \"southcentralus\", \"tags\": null, \"identity\": null, \"properties\": {\"description\": null, \"resourceId\": null, \"computeType\": \"AmlCompute\", \"computeLocation\": \"southcentralus\", \"provisioningState\": \"Succeeded\", \"provisioningErrors\": null, \"properties\": {\"vmSize\": \"STANDARD_D12_V2\", \"vmPriority\": \"LowPriority\", \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"userAccountCredentials\": {\"adminUserName\": null, \"adminUserPassword\": null, \"adminUserSshPublicKey\": null}, \"subnet\": {\"id\": null}, \"remoteLoginPortPublicAccess\": \"Enabled\"}, \"status\": {\"currentNodeCount\": 0, \"targetNodeCount\": 0, \"nodeStateCounts\": {\"preparingNodeCount\": 0, \"runningNodeCount\": 0, \"idleNodeCount\": 0, \"unusableNodeCount\": 0, \"leavingNodeCount\": 0, \"preemptedNodeCount\": 0}, \"allocationState\": \"Steady\", \"allocationStateTransitionTime\": \"2021-01-30T19:03:01.056000+00:00\", \"errors\": null, \"creationTime\": \"2021-01-30T19:02:55.373586+00:00\", \"modifiedTime\": \"2021-01-30T19:03:10.789141+00:00\", \"provisioningState\": \"Succeeded\", \"provisioningStateTransitionTime\": null, \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"vmPriority\": \"LowPriority\", \"vmSize\": \"STANDARD_D12_V2\"}}}}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"message\": \"main\", \"compute_target\": {\"id\": \"/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/computes/starts-ws-136633\", \"name\": \"starts-ws-136633\", \"location\": \"southcentralus\", \"tags\": null, \"identity\": null, \"properties\": {\"description\": null, \"resourceId\": null, \"computeType\": \"AmlCompute\", \"computeLocation\": \"southcentralus\", \"provisioningState\": \"Succeeded\", \"provisioningErrors\": null, \"properties\": {\"vmSize\": \"STANDARD_D12_V2\", \"vmPriority\": \"LowPriority\", \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"userAccountCredentials\": {\"adminUserName\": null, \"adminUserPassword\": null, \"adminUserSshPublicKey\": null}, \"subnet\": {\"id\": null}, \"remoteLoginPortPublicAccess\": \"Enabled\"}, \"status\": {\"currentNodeCount\": 0, \"targetNodeCount\": 0, \"nodeStateCounts\": {\"preparingNodeCount\": 0, \"runningNodeCount\": 0, \"idleNodeCount\": 0, \"unusableNodeCount\": 0, \"leavingNodeCount\": 0, \"preemptedNodeCount\": 0}, \"allocationState\": \"Steady\", \"allocationStateTransitionTime\": \"2021-01-30T19:03:01.056000+00:00\", \"errors\": null, \"creationTime\": \"2021-01-30T19:02:55.373586+00:00\", \"modifiedTime\": \"2021-01-30T19:03:10.789141+00:00\", \"provisioningState\": \"Succeeded\", \"provisioningStateTransitionTime\": null, \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"vmPriority\": \"LowPriority\", \"vmSize\": \"STANDARD_D12_V2\"}}}}\n",
      "{\"message\": \"main\", \"compute_target\": {\"id\": \"/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/computes/starts-ws-136633\", \"name\": \"starts-ws-136633\", \"location\": \"southcentralus\", \"tags\": null, \"identity\": null, \"properties\": {\"description\": null, \"resourceId\": null, \"computeType\": \"AmlCompute\", \"computeLocation\": \"southcentralus\", \"provisioningState\": \"Succeeded\", \"provisioningErrors\": null, \"properties\": {\"vmSize\": \"STANDARD_D12_V2\", \"vmPriority\": \"LowPriority\", \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"userAccountCredentials\": {\"adminUserName\": null, \"adminUserPassword\": null, \"adminUserSshPublicKey\": null}, \"subnet\": {\"id\": null}, \"remoteLoginPortPublicAccess\": \"Enabled\"}, \"status\": {\"currentNodeCount\": 0, \"targetNodeCount\": 0, \"nodeStateCounts\": {\"preparingNodeCount\": 0, \"runningNodeCount\": 0, \"idleNodeCount\": 0, \"unusableNodeCount\": 0, \"leavingNodeCount\": 0, \"preemptedNodeCount\": 0}, \"allocationState\": \"Steady\", \"allocationStateTransitionTime\": \"2021-01-30T19:03:01.056000+00:00\", \"errors\": null, \"creationTime\": \"2021-01-30T19:02:55.373586+00:00\", \"modifiedTime\": \"2021-01-30T19:03:10.789141+00:00\", \"provisioningState\": \"Succeeded\", \"provisioningStateTransitionTime\": null, \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"vmPriority\": \"LowPriority\", \"vmSize\": \"STANDARD_D12_V2\"}}}}\n",
      "{\"message\": \"main\", \"compute_target\": {\"id\": \"/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/computes/starts-ws-136633\", \"name\": \"starts-ws-136633\", \"location\": \"southcentralus\", \"tags\": null, \"identity\": null, \"properties\": {\"description\": null, \"resourceId\": null, \"computeType\": \"AmlCompute\", \"computeLocation\": \"southcentralus\", \"provisioningState\": \"Succeeded\", \"provisioningErrors\": null, \"properties\": {\"vmSize\": \"STANDARD_D12_V2\", \"vmPriority\": \"LowPriority\", \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"userAccountCredentials\": {\"adminUserName\": null, \"adminUserPassword\": null, \"adminUserSshPublicKey\": null}, \"subnet\": {\"id\": null}, \"remoteLoginPortPublicAccess\": \"Enabled\"}, \"status\": {\"currentNodeCount\": 0, \"targetNodeCount\": 0, \"nodeStateCounts\": {\"preparingNodeCount\": 0, \"runningNodeCount\": 0, \"idleNodeCount\": 0, \"unusableNodeCount\": 0, \"leavingNodeCount\": 0, \"preemptedNodeCount\": 0}, \"allocationState\": \"Steady\", \"allocationStateTransitionTime\": \"2021-01-30T19:03:01.056000+00:00\", \"errors\": null, \"creationTime\": \"2021-01-30T19:02:55.373586+00:00\", \"modifiedTime\": \"2021-01-30T19:03:10.789141+00:00\", \"provisioningState\": \"Succeeded\", \"provisioningStateTransitionTime\": null, \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"vmPriority\": \"LowPriority\", \"vmSize\": \"STANDARD_D12_V2\"}}}}\n",
      "{\"message\": \"main\", \"compute_target\": {\"id\": \"/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/computes/starts-ws-136633\", \"name\": \"starts-ws-136633\", \"location\": \"southcentralus\", \"tags\": null, \"identity\": null, \"properties\": {\"description\": null, \"resourceId\": null, \"computeType\": \"AmlCompute\", \"computeLocation\": \"southcentralus\", \"provisioningState\": \"Succeeded\", \"provisioningErrors\": null, \"properties\": {\"vmSize\": \"STANDARD_D12_V2\", \"vmPriority\": \"LowPriority\", \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"userAccountCredentials\": {\"adminUserName\": null, \"adminUserPassword\": null, \"adminUserSshPublicKey\": null}, \"subnet\": {\"id\": null}, \"remoteLoginPortPublicAccess\": \"Enabled\"}, \"status\": {\"currentNodeCount\": 0, \"targetNodeCount\": 0, \"nodeStateCounts\": {\"preparingNodeCount\": 0, \"runningNodeCount\": 0, \"idleNodeCount\": 0, \"unusableNodeCount\": 0, \"leavingNodeCount\": 0, \"preemptedNodeCount\": 0}, \"allocationState\": \"Steady\", \"allocationStateTransitionTime\": \"2021-01-30T19:03:01.056000+00:00\", \"errors\": null, \"creationTime\": \"2021-01-30T19:02:55.373586+00:00\", \"modifiedTime\": \"2021-01-30T19:03:10.789141+00:00\", \"provisioningState\": \"Succeeded\", \"provisioningStateTransitionTime\": null, \"scaleSettings\": {\"minNodeCount\": 0, \"maxNodeCount\": 5, \"nodeIdleTimeBeforeScaleDown\": \"PT2400S\"}, \"vmPriority\": \"LowPriority\", \"vmSize\": \"STANDARD_D12_V2\"}}}}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 726 ms, sys: 67.4 ms, total: 793 ms\n",
      "Wall time: 23.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "automl_run_config = automl_train_config.main(\n",
    "                    dataset_trainandvalidate_name=dataset_trainandvalidate_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Submit the AutoML configuration to the compute cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "gather": {
     "logged": 1598544897941
    },
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running on remote.\n",
      "CPU times: user 1.2 s, sys: 141 ms, total: 1.34 s\n",
      "Wall time: 13.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "automl_run = experiment.submit(config=automl_run_config, show_output=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Current status: DatasetEvaluation. Gathering dataset statistics.\n",
      "Current status: FeaturesGeneration. Generating features for the dataset.\n",
      "Current status: DatasetFeaturization. Beginning to fit featurizers and featurize the dataset.\n",
      "Current status: DatasetFeaturizationCompleted. Completed fit featurizers and featurizing the dataset.\n",
      "Current status: DatasetBalancing. Performing class balancing sweeping\n",
      "Current status: ModelSelection. Beginning model selection.\n",
      "\n",
      "****************************************************************************************************\n",
      "DATA GUARDRAILS: \n",
      "\n",
      "TYPE:         Class balancing detection\n",
      "STATUS:       ALERTED\n",
      "DESCRIPTION:  To decrease model bias, please cancel the current run and fix balancing problem.\n",
      "              Learn more about imbalanced data: https://aka.ms/AutomatedMLImbalancedData\n",
      "DETAILS:      Imbalanced data can lead to a falsely perceived positive effect of a model's accuracy because the input data has bias towards one class.\n",
      "+---------------------------------+---------------------------------+--------------------------------------+\n",
      "|Size of the smallest class       |Name/Label of the smallest class |Number of samples in the training data|\n",
      "+=================================+=================================+======================================+\n",
      "|31                               |SQL Injection                    |3971333                               |\n",
      "+---------------------------------+---------------------------------+--------------------------------------+\n",
      "\n",
      "****************************************************************************************************\n",
      "\n",
      "TYPE:         Missing feature values imputation\n",
      "STATUS:       PASSED\n",
      "DESCRIPTION:  No feature missing values were detected in the training data.\n",
      "              Learn more about missing value imputation: https://aka.ms/AutomatedMLFeaturization\n",
      "\n",
      "****************************************************************************************************\n",
      "\n",
      "TYPE:         High cardinality feature detection\n",
      "STATUS:       PASSED\n",
      "DESCRIPTION:  Your inputs were analyzed, and no high cardinality features were detected.\n",
      "              Learn more about high cardinality feature handling: https://aka.ms/AutomatedMLFeaturization\n",
      "\n",
      "****************************************************************************************************\n",
      "\n",
      "****************************************************************************************************\n",
      "ITERATION: The iteration being evaluated.\n",
      "PIPELINE: A summary description of the pipeline being evaluated.\n",
      "DURATION: Time taken for the current iteration.\n",
      "METRIC: The result of computing score on the fitted pipeline.\n",
      "BEST: The best observed score thus far.\n",
      "****************************************************************************************************\n",
      "\n",
      " ITERATION   PIPELINE                                       DURATION      METRIC      BEST\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"message\": \"f5366610-eee7-4f88-a3ac-adb164e5e4cc - CacheDriver:Cached token is expired at 2021-01-30 19:28:35.099456.  Refreshing\"}\n",
      "{\"message\": \"f5366610-eee7-4f88-a3ac-adb164e5e4cc - CacheDriver:Cached token is expired at 2021-01-30 19:28:35.099456.  Refreshing\"}\n",
      "{\"message\": \"f5366610-eee7-4f88-a3ac-adb164e5e4cc - CacheDriver:Cached token is expired at 2021-01-30 19:28:35.099456.  Refreshing\"}\n",
      "{\"message\": \"f5366610-eee7-4f88-a3ac-adb164e5e4cc - CacheDriver:Cached token is expired at 2021-01-30 19:28:35.099456.  Refreshing\"}\n",
      "{\"message\": \"f5366610-eee7-4f88-a3ac-adb164e5e4cc - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"f5366610-eee7-4f88-a3ac-adb164e5e4cc - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"f5366610-eee7-4f88-a3ac-adb164e5e4cc - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"f5366610-eee7-4f88-a3ac-adb164e5e4cc - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"f5366610-eee7-4f88-a3ac-adb164e5e4cc - CacheDriver:Returning token refreshed after expiry.\"}\n",
      "{\"message\": \"f5366610-eee7-4f88-a3ac-adb164e5e4cc - CacheDriver:Returning token refreshed after expiry.\"}\n",
      "{\"message\": \"f5366610-eee7-4f88-a3ac-adb164e5e4cc - CacheDriver:Returning token refreshed after expiry.\"}\n",
      "{\"message\": \"f5366610-eee7-4f88-a3ac-adb164e5e4cc - CacheDriver:Returning token refreshed after expiry.\"}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         2   MaxAbsScaler SGD                               0:07:37       0.5349    0.5349\n",
      "         3   StandardScalerWrapper XGBoostClassifier        0:09:19       0.7074    0.7074\n",
      "         0   MaxAbsScaler LightGBM                          0:09:45       0.4484    0.7074\n",
      "         6   StandardScalerWrapper LightGBM                 0:04:33       0.0000    0.7074\n",
      "         7   MaxAbsScaler LightGBM                          0:04:16       0.5254    0.7074\n",
      "         1   MaxAbsScaler XGBoostClassifier                 0:38:11       0.8223    0.8223\n",
      "         9   StandardScalerWrapper LightGBM                 0:09:45       0.8152    0.8223\n",
      "         8   StandardScalerWrapper LogisticRegression       0:17:00       0.7859    0.8223\n",
      "        12   MaxAbsScaler LightGBM                          0:05:47       0.7015    0.8223\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"message\": \"c8f63365-ff3c-45fd-84af-c087f1eac2cd - CacheDriver:Cached token is expired at 2021-01-30 20:23:39.743347.  Refreshing\"}\n",
      "{\"message\": \"c8f63365-ff3c-45fd-84af-c087f1eac2cd - CacheDriver:Cached token is expired at 2021-01-30 20:23:39.743347.  Refreshing\"}\n",
      "{\"message\": \"c8f63365-ff3c-45fd-84af-c087f1eac2cd - CacheDriver:Cached token is expired at 2021-01-30 20:23:39.743347.  Refreshing\"}\n",
      "{\"message\": \"c8f63365-ff3c-45fd-84af-c087f1eac2cd - CacheDriver:Cached token is expired at 2021-01-30 20:23:39.743347.  Refreshing\"}\n",
      "{\"message\": \"c8f63365-ff3c-45fd-84af-c087f1eac2cd - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"c8f63365-ff3c-45fd-84af-c087f1eac2cd - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"c8f63365-ff3c-45fd-84af-c087f1eac2cd - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"c8f63365-ff3c-45fd-84af-c087f1eac2cd - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"c8f63365-ff3c-45fd-84af-c087f1eac2cd - CacheDriver:Returning token refreshed after expiry.\"}\n",
      "{\"message\": \"c8f63365-ff3c-45fd-84af-c087f1eac2cd - CacheDriver:Returning token refreshed after expiry.\"}\n",
      "{\"message\": \"c8f63365-ff3c-45fd-84af-c087f1eac2cd - CacheDriver:Returning token refreshed after expiry.\"}\n",
      "{\"message\": \"c8f63365-ff3c-45fd-84af-c087f1eac2cd - CacheDriver:Returning token refreshed after expiry.\"}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        10   StandardScalerWrapper LightGBM                 0:20:18       0.8163    0.8223\n",
      "         4   StandardScalerWrapper XGBoostClassifier        1:10:48       0.8225    0.8225\n",
      "        13   StandardScalerWrapper LogisticRegression       0:26:42       0.3787    0.8225\n",
      "        11   MaxAbsScaler LogisticRegression                0:43:08          nan    0.8225\n",
      "ERROR: {\n",
      "    \"additional_properties\": {},\n",
      "    \"error\": {\n",
      "        \"additional_properties\": {\n",
      "            \"debugInfo\": null\n",
      "        },\n",
      "        \"code\": \"UserError\",\n",
      "        \"severity\": null,\n",
      "        \"message\": \"Experiment timeout reached, please consider increasing your experiment timeout.\",\n",
      "        \"message_format\": \"Experiment timeout reached, please consider increasing your experiment timeout.\",\n",
      "        \"message_parameters\": {},\n",
      "        \"reference_code\": null,\n",
      "        \"details_uri\": null,\n",
      "        \"target\": null,\n",
      "        \"details\": [],\n",
      "        \"inner_error\": {\n",
      "            \"additional_properties\": {},\n",
      "            \"code\": \"ResourceExhausted\",\n",
      "            \"inner_error\": {\n",
      "                \"additional_properties\": {},\n",
      "                \"code\": \"Timeout\",\n",
      "                \"inner_error\": {\n",
      "                    \"additional_properties\": {},\n",
      "                    \"code\": \"ExperimentTimeoutForIterations\",\n",
      "                    \"inner_error\": null\n",
      "                }\n",
      "            }\n",
      "        }\n",
      "    },\n",
      "    \"correlation\": null,\n",
      "    \"environment\": null,\n",
      "    \"location\": null,\n",
      "    \"time\": {},\n",
      "    \"component_name\": null\n",
      "}\n",
      "         5   StandardScalerWrapper XGBoostClassifier        1:23:18          nan    0.8225\n",
      "ERROR: {\n",
      "    \"additional_properties\": {},\n",
      "    \"error\": {\n",
      "        \"additional_properties\": {\n",
      "            \"debugInfo\": null\n",
      "        },\n",
      "        \"code\": \"UserError\",\n",
      "        \"severity\": null,\n",
      "        \"message\": \"Experiment timeout reached, please consider increasing your experiment timeout.\",\n",
      "        \"message_format\": \"Experiment timeout reached, please consider increasing your experiment timeout.\",\n",
      "        \"message_parameters\": {},\n",
      "        \"reference_code\": null,\n",
      "        \"details_uri\": null,\n",
      "        \"target\": null,\n",
      "        \"details\": [],\n",
      "        \"inner_error\": {\n",
      "            \"additional_properties\": {},\n",
      "            \"code\": \"ResourceExhausted\",\n",
      "            \"inner_error\": {\n",
      "                \"additional_properties\": {},\n",
      "                \"code\": \"Timeout\",\n",
      "                \"inner_error\": {\n",
      "                    \"additional_properties\": {},\n",
      "                    \"code\": \"ExperimentTimeoutForIterations\",\n",
      "                    \"inner_error\": null\n",
      "                }\n",
      "            }\n",
      "        }\n",
      "    },\n",
      "    \"correlation\": null,\n",
      "    \"environment\": null,\n",
      "    \"location\": null,\n",
      "    \"time\": {},\n",
      "    \"component_name\": null\n",
      "}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"message\": \"018ee35a-3d54-4858-a927-705f8dba60d2 - CacheDriver:Cached token is expired at 2021-01-30 21:18:43.240201.  Refreshing\"}\n",
      "{\"message\": \"018ee35a-3d54-4858-a927-705f8dba60d2 - CacheDriver:Cached token is expired at 2021-01-30 21:18:43.240201.  Refreshing\"}\n",
      "{\"message\": \"018ee35a-3d54-4858-a927-705f8dba60d2 - CacheDriver:Cached token is expired at 2021-01-30 21:18:43.240201.  Refreshing\"}\n",
      "{\"message\": \"018ee35a-3d54-4858-a927-705f8dba60d2 - CacheDriver:Cached token is expired at 2021-01-30 21:18:43.240201.  Refreshing\"}\n",
      "{\"message\": \"018ee35a-3d54-4858-a927-705f8dba60d2 - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"018ee35a-3d54-4858-a927-705f8dba60d2 - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"018ee35a-3d54-4858-a927-705f8dba60d2 - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"018ee35a-3d54-4858-a927-705f8dba60d2 - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"018ee35a-3d54-4858-a927-705f8dba60d2 - CacheDriver:Returning token refreshed after expiry.\"}\n",
      "{\"message\": \"018ee35a-3d54-4858-a927-705f8dba60d2 - CacheDriver:Returning token refreshed after expiry.\"}\n",
      "{\"message\": \"018ee35a-3d54-4858-a927-705f8dba60d2 - CacheDriver:Returning token refreshed after expiry.\"}\n",
      "{\"message\": \"018ee35a-3d54-4858-a927-705f8dba60d2 - CacheDriver:Returning token refreshed after expiry.\"}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        14    VotingEnsemble                                0:28:37       0.8347    0.8347\n",
      "CPU times: user 1min 41s, sys: 6.45 s, total: 1min 47s\n",
      "Wall time: 2h 15min 21s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'runId': 'AutoML_ae1d998c-ec87-447d-bf70-328476f80063',\n",
       " 'target': 'starts-ws-136633',\n",
       " 'status': 'Completed',\n",
       " 'startTimeUtc': '2021-01-30T19:03:45.434125Z',\n",
       " 'endTimeUtc': '2021-01-30T21:18:15.618462Z',\n",
       " 'properties': {'num_iterations': '15',\n",
       "  'training_type': 'TrainFull',\n",
       "  'acquisition_function': 'EI',\n",
       "  'primary_metric': 'norm_macro_recall',\n",
       "  'train_split': '0.3',\n",
       "  'acquisition_parameter': '0',\n",
       "  'num_cross_validation': None,\n",
       "  'target': 'starts-ws-136633',\n",
       "  'AMLSettingsJsonString': '{\"path\":null,\"name\":\"jupyter-2021-01-29-09-02\",\"subscription_id\":\"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\",\"resource_group\":\"aml-quickstarts-136633\",\"workspace_name\":\"quick-starts-ws-136633\",\"region\":\"southcentralus\",\"compute_target\":\"starts-ws-136633\",\"spark_service\":null,\"azure_service\":\"remote\",\"many_models\":false,\"pipeline_fetch_max_batch_size\":1,\"iterations\":15,\"primary_metric\":\"norm_macro_recall\",\"task_type\":\"classification\",\"data_script\":null,\"validation_size\":0.3,\"n_cross_validations\":null,\"y_min\":null,\"y_max\":null,\"num_classes\":null,\"featurization\":\"auto\",\"_ignore_package_version_incompatibilities\":false,\"is_timeseries\":false,\"max_cores_per_iteration\":-1,\"max_concurrent_iterations\":5,\"iteration_timeout_minutes\":null,\"mem_in_mb\":null,\"enforce_time_on_windows\":false,\"experiment_timeout_minutes\":90,\"experiment_exit_score\":null,\"whitelist_models\":[\"SGD\",\"LogisticRegression\",\"LightGBM\",\"XGBoostClassifier\"],\"blacklist_algos\":[\"TensorFlowLinearClassifier\",\"TensorFlowDNN\"],\"supported_models\":[\"MultinomialNaiveBayes\",\"LogisticRegression\",\"RandomForest\",\"TensorFlowDNN\",\"ExtremeRandomTrees\",\"LinearSVM\",\"AveragedPerceptronClassifier\",\"DecisionTree\",\"SVM\",\"KNN\",\"XGBoostClassifier\",\"TensorFlowLinearClassifier\",\"GradientBoosting\",\"SGD\",\"LightGBM\",\"BernoulliNaiveBayes\"],\"auto_blacklist\":true,\"blacklist_samples_reached\":false,\"exclude_nan_labels\":true,\"verbosity\":20,\"_debug_log\":\"azureml_automl.log\",\"show_warnings\":false,\"model_explainability\":true,\"service_url\":null,\"sdk_url\":null,\"sdk_packages\":null,\"enable_onnx_compatible_models\":false,\"enable_split_onnx_featurizer_estimator_models\":false,\"vm_type\":\"STANDARD_D12_V2\",\"telemetry_verbosity\":20,\"send_telemetry\":true,\"enable_dnn\":false,\"scenario\":\"SDK-1.13.0\",\"environment_label\":null,\"force_text_dnn\":false,\"enable_feature_sweeping\":true,\"enable_early_stopping\":false,\"early_stopping_n_iters\":10,\"metrics\":null,\"enable_ensembling\":true,\"enable_stack_ensembling\":false,\"ensemble_iterations\":15,\"enable_tf\":false,\"enable_subsampling\":false,\"subsample_seed\":null,\"enable_nimbusml\":false,\"enable_streaming\":false,\"force_streaming\":false,\"track_child_runs\":true,\"allowed_private_models\":[],\"label_column_name\":\"Label\",\"weight_column_name\":null,\"cv_split_column_names\":null,\"enable_local_managed\":false,\"_local_managed_run_id\":null,\"cost_mode\":1,\"lag_length\":0,\"metric_operation\":\"maximize\",\"preprocess\":true}',\n",
       "  'DataPrepJsonString': '{\\\\\"training_data\\\\\": \\\\\"{\\\\\\\\\\\\\"blocks\\\\\\\\\\\\\": [{\\\\\\\\\\\\\"id\\\\\\\\\\\\\": \\\\\\\\\\\\\"325df74e-4037-4521-b17b-305c4262518f\\\\\\\\\\\\\", \\\\\\\\\\\\\"type\\\\\\\\\\\\\": \\\\\\\\\\\\\"Microsoft.DPrep.GetDatastoreFilesBlock\\\\\\\\\\\\\", \\\\\\\\\\\\\"arguments\\\\\\\\\\\\\": {\\\\\\\\\\\\\"datastores\\\\\\\\\\\\\": [{\\\\\\\\\\\\\"datastoreName\\\\\\\\\\\\\": \\\\\\\\\\\\\"workspaceblobstore\\\\\\\\\\\\\", \\\\\\\\\\\\\"path\\\\\\\\\\\\\": \\\\\\\\\\\\\"ids2018trainandvalidate_1/data.csv\\\\\\\\\\\\\", \\\\\\\\\\\\\"resourceGroup\\\\\\\\\\\\\": \\\\\\\\\\\\\"aml-quickstarts-136633\\\\\\\\\\\\\", \\\\\\\\\\\\\"subscription\\\\\\\\\\\\\": \\\\\\\\\\\\\"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\\\\\\\\\\\\\", \\\\\\\\\\\\\"workspaceName\\\\\\\\\\\\\": \\\\\\\\\\\\\"quick-starts-ws-136633\\\\\\\\\\\\\"}]}, \\\\\\\\\\\\\"localData\\\\\\\\\\\\\": {}, \\\\\\\\\\\\\"isEnabled\\\\\\\\\\\\\": true, \\\\\\\\\\\\\"name\\\\\\\\\\\\\": null, \\\\\\\\\\\\\"annotation\\\\\\\\\\\\\": null}, {\\\\\\\\\\\\\"id\\\\\\\\\\\\\": \\\\\\\\\\\\\"77508469-4fec-485e-9faa-9026814b93ed\\\\\\\\\\\\\", \\\\\\\\\\\\\"type\\\\\\\\\\\\\": \\\\\\\\\\\\\"Microsoft.DPrep.ParseDelimitedBlock\\\\\\\\\\\\\", \\\\\\\\\\\\\"arguments\\\\\\\\\\\\\": {\\\\\\\\\\\\\"columnHeadersMode\\\\\\\\\\\\\": 3, \\\\\\\\\\\\\"fileEncoding\\\\\\\\\\\\\": 0, \\\\\\\\\\\\\"handleQuotedLineBreaks\\\\\\\\\\\\\": false, \\\\\\\\\\\\\"preview\\\\\\\\\\\\\": false, \\\\\\\\\\\\\"separator\\\\\\\\\\\\\": \\\\\\\\\\\\\",\\\\\\\\\\\\\", \\\\\\\\\\\\\"skipRows\\\\\\\\\\\\\": 0, \\\\\\\\\\\\\"skipRowsMode\\\\\\\\\\\\\": 0}, \\\\\\\\\\\\\"localData\\\\\\\\\\\\\": {}, \\\\\\\\\\\\\"isEnabled\\\\\\\\\\\\\": true, \\\\\\\\\\\\\"name\\\\\\\\\\\\\": null, \\\\\\\\\\\\\"annotation\\\\\\\\\\\\\": null}, {\\\\\\\\\\\\\"id\\\\\\\\\\\\\": \\\\\\\\\\\\\"cf15686a-b6d1-450b-8ada-338383490789\\\\\\\\\\\\\", \\\\\\\\\\\\\"type\\\\\\\\\\\\\": \\\\\\\\\\\\\"Microsoft.DPrep.DropColumnsBlock\\\\\\\\\\\\\", \\\\\\\\\\\\\"arguments\\\\\\\\\\\\\": {\\\\\\\\\\\\\"columns\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 0, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumns\\\\\\\\\\\\\": [\\\\\\\\\\\\\"Path\\\\\\\\\\\\\"]}}}, \\\\\\\\\\\\\"localData\\\\\\\\\\\\\": {}, \\\\\\\\\\\\\"isEnabled\\\\\\\\\\\\\": true, \\\\\\\\\\\\\"name\\\\\\\\\\\\\": null, \\\\\\\\\\\\\"annotation\\\\\\\\\\\\\": null}, {\\\\\\\\\\\\\"id\\\\\\\\\\\\\": \\\\\\\\\\\\\"843070be-33dd-4bb8-a164-d019c244df94\\\\\\\\\\\\\", \\\\\\\\\\\\\"type\\\\\\\\\\\\\": \\\\\\\\\\\\\"Microsoft.DPrep.SetColumnTypesBlock\\\\\\\\\\\\\", \\\\\\\\\\\\\"arguments\\\\\\\\\\\\\": {\\\\\\\\\\\\\"columnConversion\\\\\\\\\\\\\": [{\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Flow Duration\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"TotLen Fwd Pkts\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"TotLen Bwd Pkts\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Fwd Pkt Len Std\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Bwd Pkt Len Max\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Bwd Pkt Len Std\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Flow Byts/s\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Flow Pkts/s\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Flow IAT Max\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Bwd IAT Min\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Bwd Header Len\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Pkt Len Max\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Pkt Len Std\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"RST Flag Cnt\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"PSH Flag Cnt\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"ECE Flag Cnt\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Init Fwd Win Byts\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Init Bwd Win Byts\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Fwd Seg Size Min\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Label\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 0}]}, \\\\\\\\\\\\\"localData\\\\\\\\\\\\\": {}, \\\\\\\\\\\\\"isEnabled\\\\\\\\\\\\\": true, \\\\\\\\\\\\\"name\\\\\\\\\\\\\": null, \\\\\\\\\\\\\"annotation\\\\\\\\\\\\\": null}, {\\\\\\\\\\\\\"id\\\\\\\\\\\\\": \\\\\\\\\\\\\"63040661-70e7-473d-b974-a3499aaec589\\\\\\\\\\\\\", \\\\\\\\\\\\\"type\\\\\\\\\\\\\": \\\\\\\\\\\\\"Microsoft.DPrep.SetColumnTypesBlock\\\\\\\\\\\\\", \\\\\\\\\\\\\"arguments\\\\\\\\\\\\\": {\\\\\\\\\\\\\"columnConversion\\\\\\\\\\\\\": [{\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Flow Duration\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"TotLen Fwd Pkts\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"TotLen Bwd Pkts\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Fwd Pkt Len Std\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Bwd Pkt Len Max\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Bwd Pkt Len Std\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Flow Byts/s\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Flow Pkts/s\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Flow IAT Max\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Bwd IAT Min\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Bwd Header Len\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Pkt Len Max\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Pkt Len Std\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"RST Flag Cnt\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"PSH Flag Cnt\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"ECE Flag Cnt\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Init Fwd Win Byts\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Init Bwd Win Byts\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Fwd Seg Size Min\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\"column\\\\\\\\\\\\\": {\\\\\\\\\\\\\"type\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\"details\\\\\\\\\\\\\": {\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\": \\\\\\\\\\\\\"Label\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\": 0}]}, \\\\\\\\\\\\\"localData\\\\\\\\\\\\\": {}, \\\\\\\\\\\\\"isEnabled\\\\\\\\\\\\\": true, \\\\\\\\\\\\\"name\\\\\\\\\\\\\": null, \\\\\\\\\\\\\"annotation\\\\\\\\\\\\\": null}], \\\\\\\\\\\\\"inspectors\\\\\\\\\\\\\": [], \\\\\\\\\\\\\"meta\\\\\\\\\\\\\": {\\\\\\\\\\\\\"savedDatasetId\\\\\\\\\\\\\": \\\\\\\\\\\\\"1996787f-c614-49ab-9aa8-9ed33ee4536d\\\\\\\\\\\\\", \\\\\\\\\\\\\"datasetType\\\\\\\\\\\\\": \\\\\\\\\\\\\"tabular\\\\\\\\\\\\\", \\\\\\\\\\\\\"subscriptionId\\\\\\\\\\\\\": \\\\\\\\\\\\\"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\\\\\\\\\\\\\", \\\\\\\\\\\\\"workspaceId\\\\\\\\\\\\\": \\\\\\\\\\\\\"22db85a2-8b83-4c9c-9b6b-7fa5c0318891\\\\\\\\\\\\\", \\\\\\\\\\\\\"workspaceLocation\\\\\\\\\\\\\": \\\\\\\\\\\\\"southcentralus\\\\\\\\\\\\\"}}\\\\\", \\\\\"activities\\\\\": 0}',\n",
       "  'EnableSubsampling': 'False',\n",
       "  'runTemplate': 'AutoML',\n",
       "  'azureml.runsource': 'automl',\n",
       "  'display_task_type': 'classification',\n",
       "  'dependencies_versions': '{\"azureml-widgets\": \"1.21.0\", \"azureml-train\": \"1.21.0\", \"azureml-train-restclients-hyperdrive\": \"1.21.0\", \"azureml-train-core\": \"1.21.0\", \"azureml-train-automl\": \"1.21.0\", \"azureml-train-automl-runtime\": \"1.21.0.post1\", \"azureml-train-automl-client\": \"1.21.0\", \"azureml-telemetry\": \"1.21.0\", \"azureml-sdk\": \"1.21.0\", \"azureml-pipeline\": \"1.21.0\", \"azureml-pipeline-steps\": \"1.21.0\", \"azureml-pipeline-core\": \"1.21.0\", \"azureml-opendatasets\": \"1.21.0\", \"azureml-model-management-sdk\": \"1.0.1b6.post1\", \"azureml-interpret\": \"1.21.0\", \"azureml-defaults\": \"1.21.0\", \"azureml-dataset-runtime\": \"1.21.0\", \"azureml-dataprep\": \"2.8.2\", \"azureml-dataprep-rslex\": \"1.6.0\", \"azureml-dataprep-native\": \"28.0.0\", \"azureml-core\": \"1.21.0.post1\", \"azureml-automl-runtime\": \"1.21.0\", \"azureml-automl-core\": \"1.21.0\"}',\n",
       "  '_aml_system_scenario_identification': 'Remote.Parent',\n",
       "  'ClientType': 'SDK',\n",
       "  'environment_cpu_name': 'AzureML-AutoML',\n",
       "  'environment_cpu_label': 'prod',\n",
       "  'environment_gpu_name': 'AzureML-AutoML-GPU',\n",
       "  'environment_gpu_label': 'prod',\n",
       "  'root_attribution': 'automl',\n",
       "  'attribution': 'AutoML',\n",
       "  'Orchestrator': 'AutoML',\n",
       "  'CancelUri': 'https://southcentralus.experiments.azureml.net/jasmine/v1.0/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experimentids/8ce9f2e2-bb64-4a98-9d31-ce63446013df/cancel/AutoML_ae1d998c-ec87-447d-bf70-328476f80063',\n",
       "  'ClientSdkVersion': '1.20.0.post1',\n",
       "  'snapshotId': '00000000-0000-0000-0000-000000000000',\n",
       "  'SetupRunId': 'AutoML_ae1d998c-ec87-447d-bf70-328476f80063_setup',\n",
       "  'SetupRunContainerId': 'dcid.AutoML_ae1d998c-ec87-447d-bf70-328476f80063_setup',\n",
       "  'FeaturizationRunJsonPath': 'featurizer_container.json',\n",
       "  'FeaturizationRunId': 'AutoML_ae1d998c-ec87-447d-bf70-328476f80063_featurize',\n",
       "  'ProblemInfoJsonString': '{\"dataset_num_categorical\": 0, \"is_sparse\": true, \"subsampling\": false, \"dataset_classes\": 14, \"dataset_features\": 28, \"dataset_samples\": 3971333, \"single_frequency_class_detected\": false}',\n",
       "  'ModelExplainRunId': 'AutoML_ae1d998c-ec87-447d-bf70-328476f80063_ModelExplain'},\n",
       " 'inputDatasets': [{'dataset': {'id': '1996787f-c614-49ab-9aa8-9ed33ee4536d'}, 'consumptionDetails': {'type': 'RunInput', 'inputName': 'training_data', 'mechanism': 'Direct'}}],\n",
       " 'outputDatasets': [],\n",
       " 'logFiles': {},\n",
       " 'submittedBy': 'ODL_User 136633'}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "automl_run.wait_for_completion(show_output=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert(automl_run.get_status() == \"Completed\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "gather": {
     "logged": 1598544898497
    },
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    }
   },
   "source": [
    "## Run Details"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The widget below shows the first few best runs of the experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "gather": {
     "logged": 1598546648408
    },
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "711ddaeb36244169bc27e2477d9a1b6b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "_AutoMLWidget(widget_settings={'childWidgetDisplay': 'popup', 'send_telemetry': False, 'log_level': 'INFO', 's…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/aml.mini.widget.v1": "{\"status\": \"Completed\", \"workbench_run_details_uri\": \"https://ml.azure.com/experiments/jupyter-2021-01-29-09-02/runs/AutoML_ae1d998c-ec87-447d-bf70-328476f80063?wsid=/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourcegroups/aml-quickstarts-136633/workspaces/quick-starts-ws-136633\", \"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"run_properties\": {\"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"created_utc\": \"2021-01-30T19:03:27.174404Z\", \"properties\": {\"num_iterations\": \"15\", \"training_type\": \"TrainFull\", \"acquisition_function\": \"EI\", \"primary_metric\": \"norm_macro_recall\", \"train_split\": \"0.3\", \"acquisition_parameter\": \"0\", \"num_cross_validation\": null, \"target\": \"starts-ws-136633\", \"AMLSettingsJsonString\": \"{\\\"path\\\":null,\\\"name\\\":\\\"jupyter-2021-01-29-09-02\\\",\\\"subscription_id\\\":\\\"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\\\",\\\"resource_group\\\":\\\"aml-quickstarts-136633\\\",\\\"workspace_name\\\":\\\"quick-starts-ws-136633\\\",\\\"region\\\":\\\"southcentralus\\\",\\\"compute_target\\\":\\\"starts-ws-136633\\\",\\\"spark_service\\\":null,\\\"azure_service\\\":\\\"remote\\\",\\\"many_models\\\":false,\\\"pipeline_fetch_max_batch_size\\\":1,\\\"iterations\\\":15,\\\"primary_metric\\\":\\\"norm_macro_recall\\\",\\\"task_type\\\":\\\"classification\\\",\\\"data_script\\\":null,\\\"validation_size\\\":0.3,\\\"n_cross_validations\\\":null,\\\"y_min\\\":null,\\\"y_max\\\":null,\\\"num_classes\\\":null,\\\"featurization\\\":\\\"auto\\\",\\\"_ignore_package_version_incompatibilities\\\":false,\\\"is_timeseries\\\":false,\\\"max_cores_per_iteration\\\":-1,\\\"max_concurrent_iterations\\\":5,\\\"iteration_timeout_minutes\\\":null,\\\"mem_in_mb\\\":null,\\\"enforce_time_on_windows\\\":false,\\\"experiment_timeout_minutes\\\":90,\\\"experiment_exit_score\\\":null,\\\"whitelist_models\\\":[\\\"SGD\\\",\\\"LogisticRegression\\\",\\\"LightGBM\\\",\\\"XGBoostClassifier\\\"],\\\"blacklist_algos\\\":[\\\"TensorFlowLinearClassifier\\\",\\\"TensorFlowDNN\\\"],\\\"supported_models\\\":[\\\"MultinomialNaiveBayes\\\",\\\"LogisticRegression\\\",\\\"RandomForest\\\",\\\"TensorFlowDNN\\\",\\\"ExtremeRandomTrees\\\",\\\"LinearSVM\\\",\\\"AveragedPerceptronClassifier\\\",\\\"DecisionTree\\\",\\\"SVM\\\",\\\"KNN\\\",\\\"XGBoostClassifier\\\",\\\"TensorFlowLinearClassifier\\\",\\\"GradientBoosting\\\",\\\"SGD\\\",\\\"LightGBM\\\",\\\"BernoulliNaiveBayes\\\"],\\\"auto_blacklist\\\":true,\\\"blacklist_samples_reached\\\":false,\\\"exclude_nan_labels\\\":true,\\\"verbosity\\\":20,\\\"_debug_log\\\":\\\"azureml_automl.log\\\",\\\"show_warnings\\\":false,\\\"model_explainability\\\":true,\\\"service_url\\\":null,\\\"sdk_url\\\":null,\\\"sdk_packages\\\":null,\\\"enable_onnx_compatible_models\\\":false,\\\"enable_split_onnx_featurizer_estimator_models\\\":false,\\\"vm_type\\\":\\\"STANDARD_D12_V2\\\",\\\"telemetry_verbosity\\\":20,\\\"send_telemetry\\\":true,\\\"enable_dnn\\\":false,\\\"scenario\\\":\\\"SDK-1.13.0\\\",\\\"environment_label\\\":null,\\\"force_text_dnn\\\":false,\\\"enable_feature_sweeping\\\":true,\\\"enable_early_stopping\\\":false,\\\"early_stopping_n_iters\\\":10,\\\"metrics\\\":null,\\\"enable_ensembling\\\":true,\\\"enable_stack_ensembling\\\":false,\\\"ensemble_iterations\\\":15,\\\"enable_tf\\\":false,\\\"enable_subsampling\\\":false,\\\"subsample_seed\\\":null,\\\"enable_nimbusml\\\":false,\\\"enable_streaming\\\":false,\\\"force_streaming\\\":false,\\\"track_child_runs\\\":true,\\\"allowed_private_models\\\":[],\\\"label_column_name\\\":\\\"Label\\\",\\\"weight_column_name\\\":null,\\\"cv_split_column_names\\\":null,\\\"enable_local_managed\\\":false,\\\"_local_managed_run_id\\\":null,\\\"cost_mode\\\":1,\\\"lag_length\\\":0,\\\"metric_operation\\\":\\\"maximize\\\",\\\"preprocess\\\":true}\", \"DataPrepJsonString\": \"{\\\\\\\"training_data\\\\\\\": \\\\\\\"{\\\\\\\\\\\\\\\"blocks\\\\\\\\\\\\\\\": [{\\\\\\\\\\\\\\\"id\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"325df74e-4037-4521-b17b-305c4262518f\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Microsoft.DPrep.GetDatastoreFilesBlock\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"arguments\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"datastores\\\\\\\\\\\\\\\": [{\\\\\\\\\\\\\\\"datastoreName\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"workspaceblobstore\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"path\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ids2018trainandvalidate_1/data.csv\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"resourceGroup\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"aml-quickstarts-136633\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"subscription\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"workspaceName\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"quick-starts-ws-136633\\\\\\\\\\\\\\\"}]}, \\\\\\\\\\\\\\\"localData\\\\\\\\\\\\\\\": {}, \\\\\\\\\\\\\\\"isEnabled\\\\\\\\\\\\\\\": true, \\\\\\\\\\\\\\\"name\\\\\\\\\\\\\\\": null, \\\\\\\\\\\\\\\"annotation\\\\\\\\\\\\\\\": null}, {\\\\\\\\\\\\\\\"id\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"77508469-4fec-485e-9faa-9026814b93ed\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Microsoft.DPrep.ParseDelimitedBlock\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"arguments\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"columnHeadersMode\\\\\\\\\\\\\\\": 3, \\\\\\\\\\\\\\\"fileEncoding\\\\\\\\\\\\\\\": 0, \\\\\\\\\\\\\\\"handleQuotedLineBreaks\\\\\\\\\\\\\\\": false, \\\\\\\\\\\\\\\"preview\\\\\\\\\\\\\\\": false, \\\\\\\\\\\\\\\"separator\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"skipRows\\\\\\\\\\\\\\\": 0, \\\\\\\\\\\\\\\"skipRowsMode\\\\\\\\\\\\\\\": 0}, \\\\\\\\\\\\\\\"localData\\\\\\\\\\\\\\\": {}, \\\\\\\\\\\\\\\"isEnabled\\\\\\\\\\\\\\\": true, \\\\\\\\\\\\\\\"name\\\\\\\\\\\\\\\": null, \\\\\\\\\\\\\\\"annotation\\\\\\\\\\\\\\\": null}, {\\\\\\\\\\\\\\\"id\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"cf15686a-b6d1-450b-8ada-338383490789\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Microsoft.DPrep.DropColumnsBlock\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"arguments\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"columns\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 0, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumns\\\\\\\\\\\\\\\": [\\\\\\\\\\\\\\\"Path\\\\\\\\\\\\\\\"]}}}, \\\\\\\\\\\\\\\"localData\\\\\\\\\\\\\\\": {}, \\\\\\\\\\\\\\\"isEnabled\\\\\\\\\\\\\\\": true, \\\\\\\\\\\\\\\"name\\\\\\\\\\\\\\\": null, \\\\\\\\\\\\\\\"annotation\\\\\\\\\\\\\\\": null}, {\\\\\\\\\\\\\\\"id\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"843070be-33dd-4bb8-a164-d019c244df94\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Microsoft.DPrep.SetColumnTypesBlock\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"arguments\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"columnConversion\\\\\\\\\\\\\\\": [{\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Flow Duration\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"TotLen Fwd Pkts\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"TotLen Bwd Pkts\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Fwd Pkt Len Std\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Bwd Pkt Len Max\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Bwd Pkt Len Std\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Flow Byts/s\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Flow Pkts/s\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Flow IAT Max\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Bwd IAT Min\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Bwd Header Len\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Pkt Len Max\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Pkt Len Std\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"RST Flag Cnt\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"PSH Flag Cnt\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ECE Flag Cnt\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Init Fwd Win Byts\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Init Bwd Win Byts\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Fwd Seg Size Min\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Label\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 0}]}, \\\\\\\\\\\\\\\"localData\\\\\\\\\\\\\\\": {}, \\\\\\\\\\\\\\\"isEnabled\\\\\\\\\\\\\\\": true, \\\\\\\\\\\\\\\"name\\\\\\\\\\\\\\\": null, \\\\\\\\\\\\\\\"annotation\\\\\\\\\\\\\\\": null}, {\\\\\\\\\\\\\\\"id\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"63040661-70e7-473d-b974-a3499aaec589\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Microsoft.DPrep.SetColumnTypesBlock\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"arguments\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"columnConversion\\\\\\\\\\\\\\\": [{\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Flow Duration\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"TotLen Fwd Pkts\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"TotLen Bwd Pkts\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Fwd Pkt Len Std\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Bwd Pkt Len Max\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Bwd Pkt Len Std\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Flow Byts/s\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Flow Pkts/s\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Flow IAT Max\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Bwd IAT Min\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Bwd Header Len\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Pkt Len Max\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Pkt Len Std\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"RST Flag Cnt\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"PSH Flag Cnt\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ECE Flag Cnt\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Init Fwd Win Byts\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Init Bwd Win Byts\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Fwd Seg Size Min\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 2}, {\\\\\\\\\\\\\\\"column\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"type\\\\\\\\\\\\\\\": 2, \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"selectedColumn\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Label\\\\\\\\\\\\\\\"}}, \\\\\\\\\\\\\\\"typeProperty\\\\\\\\\\\\\\\": 0}]}, \\\\\\\\\\\\\\\"localData\\\\\\\\\\\\\\\": {}, \\\\\\\\\\\\\\\"isEnabled\\\\\\\\\\\\\\\": true, \\\\\\\\\\\\\\\"name\\\\\\\\\\\\\\\": null, \\\\\\\\\\\\\\\"annotation\\\\\\\\\\\\\\\": null}], \\\\\\\\\\\\\\\"inspectors\\\\\\\\\\\\\\\": [], \\\\\\\\\\\\\\\"meta\\\\\\\\\\\\\\\": {\\\\\\\\\\\\\\\"savedDatasetId\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"1996787f-c614-49ab-9aa8-9ed33ee4536d\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"datasetType\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"tabular\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"subscriptionId\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"workspaceId\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"22db85a2-8b83-4c9c-9b6b-7fa5c0318891\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"workspaceLocation\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"southcentralus\\\\\\\\\\\\\\\"}}\\\\\\\", \\\\\\\"activities\\\\\\\": 0}\", \"EnableSubsampling\": \"False\", \"runTemplate\": \"AutoML\", \"azureml.runsource\": \"automl\", \"display_task_type\": \"classification\", \"dependencies_versions\": \"{\\\"azureml-widgets\\\": \\\"1.21.0\\\", \\\"azureml-train\\\": \\\"1.21.0\\\", \\\"azureml-train-restclients-hyperdrive\\\": \\\"1.21.0\\\", \\\"azureml-train-core\\\": \\\"1.21.0\\\", \\\"azureml-train-automl\\\": \\\"1.21.0\\\", \\\"azureml-train-automl-runtime\\\": \\\"1.21.0.post1\\\", \\\"azureml-train-automl-client\\\": \\\"1.21.0\\\", \\\"azureml-telemetry\\\": \\\"1.21.0\\\", \\\"azureml-sdk\\\": \\\"1.21.0\\\", \\\"azureml-pipeline\\\": \\\"1.21.0\\\", \\\"azureml-pipeline-steps\\\": \\\"1.21.0\\\", \\\"azureml-pipeline-core\\\": \\\"1.21.0\\\", \\\"azureml-opendatasets\\\": \\\"1.21.0\\\", \\\"azureml-model-management-sdk\\\": \\\"1.0.1b6.post1\\\", \\\"azureml-interpret\\\": \\\"1.21.0\\\", \\\"azureml-defaults\\\": \\\"1.21.0\\\", \\\"azureml-dataset-runtime\\\": \\\"1.21.0\\\", \\\"azureml-dataprep\\\": \\\"2.8.2\\\", \\\"azureml-dataprep-rslex\\\": \\\"1.6.0\\\", \\\"azureml-dataprep-native\\\": \\\"28.0.0\\\", \\\"azureml-core\\\": \\\"1.21.0.post1\\\", \\\"azureml-automl-runtime\\\": \\\"1.21.0\\\", \\\"azureml-automl-core\\\": \\\"1.21.0\\\"}\", \"_aml_system_scenario_identification\": \"Remote.Parent\", \"ClientType\": \"SDK\", \"environment_cpu_name\": \"AzureML-AutoML\", \"environment_cpu_label\": \"prod\", \"environment_gpu_name\": \"AzureML-AutoML-GPU\", \"environment_gpu_label\": \"prod\", \"root_attribution\": \"automl\", \"attribution\": \"AutoML\", \"Orchestrator\": \"AutoML\", \"CancelUri\": \"https://southcentralus.experiments.azureml.net/jasmine/v1.0/subscriptions/a24a24d5-8d87-4c8a-99b6-91ed2d2df51f/resourceGroups/aml-quickstarts-136633/providers/Microsoft.MachineLearningServices/workspaces/quick-starts-ws-136633/experimentids/8ce9f2e2-bb64-4a98-9d31-ce63446013df/cancel/AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"ClientSdkVersion\": \"1.20.0.post1\", \"snapshotId\": \"00000000-0000-0000-0000-000000000000\", \"SetupRunId\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063_setup\", \"SetupRunContainerId\": \"dcid.AutoML_ae1d998c-ec87-447d-bf70-328476f80063_setup\", \"FeaturizationRunJsonPath\": \"featurizer_container.json\", \"FeaturizationRunId\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063_featurize\", \"ProblemInfoJsonString\": \"{\\\"dataset_num_categorical\\\": 0, \\\"is_sparse\\\": true, \\\"subsampling\\\": false, \\\"dataset_classes\\\": 14, \\\"dataset_features\\\": 28, \\\"dataset_samples\\\": 3971333, \\\"single_frequency_class_detected\\\": false}\", \"ModelExplainRunId\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063_ModelExplain\"}, \"tags\": {\"model_explain_run\": \"best_run\", \"_aml_system_azureml.automlComponent\": \"AutoML\", \"pipeline_id\": \"\", \"score\": \"\", \"predicted_cost\": \"\", \"fit_time\": \"\", \"training_percent\": \"\", \"iteration\": \"\", \"run_preprocessor\": \"\", \"run_algorithm\": \"\", \"automl_best_child_run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14\", \"model_explain_best_run_child_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14\"}, \"end_time_utc\": \"2021-01-30T21:18:15.618462Z\", \"status\": \"Completed\", \"log_files\": {}, \"log_groups\": [], \"run_duration\": \"2:14:48\", \"run_number\": \"1\", \"run_queued_details\": {\"status\": \"Completed\", \"details\": null}}, \"child_runs\": [{\"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063_0\", \"run_number\": 5, \"metric\": null, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": \"100\", \"start_time\": \"2021-01-30T19:20:51.694651Z\", \"end_time\": \"2021-01-30T19:30:36.287106Z\", \"created_time\": \"2021-01-30T19:17:53.707023Z\", \"created_time_dt\": \"2021-01-30T19:17:53.707023Z\", \"duration\": \"0:12:42\", \"iteration\": \"0\", \"goal\": \"norm_macro_recall_max\", \"run_name\": \"MaxAbsScaler, LightGBM\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.44837046, \"best_metric\": 0.44837046}, {\"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063_2\", \"run_number\": 6, \"metric\": null, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": \"100\", \"start_time\": \"2021-01-30T19:18:04.157302Z\", \"end_time\": \"2021-01-30T19:25:40.705122Z\", \"created_time\": \"2021-01-30T19:17:53.722322Z\", \"created_time_dt\": \"2021-01-30T19:17:53.722322Z\", \"duration\": \"0:07:46\", \"iteration\": \"2\", \"goal\": \"norm_macro_recall_max\", \"run_name\": \"MaxAbsScaler, SGD\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.53487887, \"best_metric\": 0.82227906}, {\"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063_3\", \"run_number\": 7, \"metric\": null, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": \"100\", \"start_time\": \"2021-01-30T19:20:49.454657Z\", \"end_time\": \"2021-01-30T19:30:08.161932Z\", \"created_time\": \"2021-01-30T19:17:53.733484Z\", \"created_time_dt\": \"2021-01-30T19:17:53.733484Z\", \"duration\": \"0:12:14\", \"iteration\": \"3\", \"goal\": \"norm_macro_recall_max\", \"run_name\": \"StandardScalerWrapper, XGBoostClassifier\", \"run_properties\": \"<azureml.automl.runtime.shared.model_wrappers.StandardScalerWrapper object at 0x7f5cf1bca780\", \"primary_metric\": 0.70742125, \"best_metric\": 0.82227906}, {\"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063_1\", \"run_number\": 8, \"metric\": null, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": \"100\", \"start_time\": \"2021-01-30T19:20:48.567407Z\", \"end_time\": \"2021-01-30T19:58:59.53615Z\", \"created_time\": \"2021-01-30T19:17:53.75309Z\", \"created_time_dt\": \"2021-01-30T19:17:53.75309Z\", \"duration\": \"0:41:05\", \"iteration\": \"1\", \"goal\": \"norm_macro_recall_max\", \"run_name\": \"MaxAbsScaler, XGBoostClassifier\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.82227906, \"best_metric\": 0.82227906}, {\"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063_4\", \"run_number\": 9, \"metric\": null, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": \"100\", \"start_time\": \"2021-01-30T19:20:49.116112Z\", \"end_time\": \"2021-01-30T20:31:37.072391Z\", \"created_time\": \"2021-01-30T19:17:53.728068Z\", \"created_time_dt\": \"2021-01-30T19:17:53.728068Z\", \"duration\": \"1:13:43\", \"iteration\": \"4\", \"goal\": \"norm_macro_recall_max\", \"run_name\": \"StandardScalerWrapper, XGBoostClassifier\", \"run_properties\": \"<azureml.automl.runtime.shared.model_wrappers.StandardScalerWrapper object at 0x7fb8ae9bee10\", \"primary_metric\": 0.82248973, \"best_metric\": 0.82248973}, {\"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063_5\", \"run_number\": 10, \"metric\": null, \"status\": \"Canceled\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": \"100\", \"start_time\": \"2021-01-30T19:25:56.471439Z\", \"end_time\": \"2021-01-30T20:49:14.711072Z\", \"created_time\": \"2021-01-30T19:25:43.703577Z\", \"created_time_dt\": \"2021-01-30T19:25:43.703577Z\", \"duration\": \"1:23:31\", \"iteration\": \"5\", \"goal\": null, \"run_name\": \"StandardScalerWrapper, XGBoostClassifier\", \"run_properties\": null}, {\"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063_6\", \"run_number\": 11, \"metric\": null, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": \"100\", \"start_time\": \"2021-01-30T19:38:14.474433Z\", \"end_time\": \"2021-01-30T19:42:47.040116Z\", \"created_time\": \"2021-01-30T19:30:12.909164Z\", \"created_time_dt\": \"2021-01-30T19:30:12.909164Z\", \"duration\": \"0:12:34\", \"iteration\": \"6\", \"goal\": \"norm_macro_recall_max\", \"run_name\": \"StandardScalerWrapper, LightGBM\", \"run_properties\": \"<azureml.automl.runtime.shared.model_wrappers.StandardScalerWrapper object at 0x7f768a2c0be0\", \"primary_metric\": 0.0, \"best_metric\": 0.82248973}, {\"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063_7\", \"run_number\": 12, \"metric\": null, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": \"100\", \"start_time\": \"2021-01-30T19:40:20.280158Z\", \"end_time\": \"2021-01-30T19:44:36.169158Z\", \"created_time\": \"2021-01-30T19:30:39.49943Z\", \"created_time_dt\": \"2021-01-30T19:30:39.49943Z\", \"duration\": \"0:13:56\", \"iteration\": \"7\", \"goal\": \"norm_macro_recall_max\", \"run_name\": \"MaxAbsScaler, LightGBM\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.52538956, \"best_metric\": 0.82248973}, {\"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063_8\", \"run_number\": 13, \"metric\": null, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": \"100\", \"start_time\": \"2021-01-30T19:50:59.335143Z\", \"end_time\": \"2021-01-30T20:07:58.844353Z\", \"created_time\": \"2021-01-30T19:42:49.579734Z\", \"created_time_dt\": \"2021-01-30T19:42:49.579734Z\", \"duration\": \"0:25:09\", \"iteration\": \"8\", \"goal\": \"norm_macro_recall_max\", \"run_name\": \"StandardScalerWrapper, LogisticRegression\", \"run_properties\": \"<azureml.automl.runtime.shared.model_wrappers.StandardScalerWrapper object at 0x7f83d078dc18\", \"primary_metric\": 0.7859391, \"best_metric\": 0.82248973}, {\"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063_9\", \"run_number\": 14, \"metric\": null, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": \"100\", \"start_time\": \"2021-01-30T19:52:30.987889Z\", \"end_time\": \"2021-01-30T20:02:16.185071Z\", \"created_time\": \"2021-01-30T19:44:38.680346Z\", \"created_time_dt\": \"2021-01-30T19:44:38.680346Z\", \"duration\": \"0:17:37\", \"iteration\": \"9\", \"goal\": \"norm_macro_recall_max\", \"run_name\": \"StandardScalerWrapper, LightGBM\", \"run_properties\": \"<azureml.automl.runtime.shared.model_wrappers.StandardScalerWrapper object at 0x7f34b7474a58\", \"primary_metric\": 0.81522118, \"best_metric\": 0.82248973}, {\"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063_10\", \"run_number\": 15, \"metric\": null, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": \"100\", \"start_time\": \"2021-01-30T20:02:56.8901Z\", \"end_time\": \"2021-01-30T20:23:14.995441Z\", \"created_time\": \"2021-01-30T19:59:01.759491Z\", \"created_time_dt\": \"2021-01-30T19:59:01.759491Z\", \"duration\": \"0:24:13\", \"iteration\": \"10\", \"goal\": \"norm_macro_recall_max\", \"run_name\": \"StandardScalerWrapper, LightGBM\", \"run_properties\": \"<azureml.automl.runtime.shared.model_wrappers.StandardScalerWrapper object at 0x7f70120929b0\", \"primary_metric\": 0.81629917, \"best_metric\": 0.82248973}, {\"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063_11\", \"run_number\": 16, \"metric\": null, \"status\": \"Canceled\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": \"100\", \"start_time\": \"2021-01-30T20:05:49.746208Z\", \"end_time\": \"2021-01-30T20:48:58.142975Z\", \"created_time\": \"2021-01-30T20:02:18.461042Z\", \"created_time_dt\": \"2021-01-30T20:02:18.461042Z\", \"duration\": \"0:46:39\", \"iteration\": \"11\", \"goal\": null, \"run_name\": \"MaxAbsScaler, LogisticRegression\", \"run_properties\": null}, {\"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063_12\", \"run_number\": 17, \"metric\": null, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": \"100\", \"start_time\": \"2021-01-30T20:08:11.583859Z\", \"end_time\": \"2021-01-30T20:13:58.563114Z\", \"created_time\": \"2021-01-30T20:08:01.157022Z\", \"created_time_dt\": \"2021-01-30T20:08:01.157022Z\", \"duration\": \"0:05:57\", \"iteration\": \"12\", \"goal\": \"norm_macro_recall_max\", \"run_name\": \"MaxAbsScaler, LightGBM\", \"run_properties\": \"copy=True\", \"primary_metric\": 0.70152709, \"best_metric\": 0.82248973}, {\"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063_13\", \"run_number\": 18, \"metric\": null, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": \"100\", \"start_time\": \"2021-01-30T20:14:14.08738Z\", \"end_time\": \"2021-01-30T20:40:56.125107Z\", \"created_time\": \"2021-01-30T20:14:01.698671Z\", \"created_time_dt\": \"2021-01-30T20:14:01.698671Z\", \"duration\": \"0:26:54\", \"iteration\": \"13\", \"goal\": \"norm_macro_recall_max\", \"run_name\": \"StandardScalerWrapper, LogisticRegression\", \"run_properties\": \"<azureml.automl.runtime.shared.model_wrappers.StandardScalerWrapper object at 0x7f5470a8ac88\", \"primary_metric\": 0.37868897, \"best_metric\": 0.82248973}, {\"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14\", \"run_number\": 19, \"metric\": null, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": \"100\", \"start_time\": \"2021-01-30T20:49:34.62987Z\", \"end_time\": \"2021-01-30T21:18:11.219349Z\", \"created_time\": \"2021-01-30T20:49:17.23495Z\", \"created_time_dt\": \"2021-01-30T20:49:17.23495Z\", \"duration\": \"0:28:53\", \"iteration\": \"14\", \"goal\": \"norm_macro_recall_max\", \"run_name\": \"VotingEnsemble\", \"run_properties\": \"classification_labels=None,\\n                              estimators=[('4',\\n                                           Pipeline(memory=None,\\n                                                    steps=[('standardscalerwrapper',\\n                                                            <azureml.automl.runtime.shared.model_wrappers.StandardScalerWrapper object at 0x7f40840bd550>\", \"primary_metric\": 0.83465447, \"best_metric\": 0.83465447}], \"children_metrics\": {\"categories\": [0], \"series\": {\"log_loss\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"markers\", \"name\": \"log_loss\", \"stepped\": false, \"type\": \"scatter\", \"data\": [3.884052867029191, 0.06623514327974366, 0.6562415801285079, 0.6571319247117947, 0.06195533686899778, 0.3980014009122021, 0.21299107092464853, 0.46419019416530155, 0.06686328210720786, 0.06532032054556354, 0.09194466739143202, 0.31724249343566774, 0.22456691217636446]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"lines\", \"name\": \"log_loss_min\", \"stepped\": true, \"type\": \"scatter\", \"data\": [3.884052867029191, 0.06623514327974366, 0.06623514327974366, 0.06623514327974366, 0.06195533686899778, 0.06195533686899778, 0.06195533686899778, 0.06195533686899778, 0.06195533686899778, 0.06195533686899778, 0.06195533686899778, 0.06195533686899778, 0.06195533686899778]}], \"matthews_correlation\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"markers\", \"name\": \"matthews_correlation\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.6757937973854912, 0.917834999746962, 0.5759856340136154, 0.9170862070555397, 0.918884156605897, 0.0, 0.8894476072391945, 0.7436817181754438, 0.9173496452949329, 0.9177996636026603, 0.8963779604669325, 0.7942411451912907, 0.9060454227132824]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"lines\", \"name\": \"matthews_correlation_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.6757937973854912, 0.917834999746962, 0.917834999746962, 0.917834999746962, 0.918884156605897, 0.918884156605897, 0.918884156605897, 0.918884156605897, 0.918884156605897, 0.918884156605897, 0.918884156605897, 0.918884156605897, 0.918884156605897]}], \"average_precision_score_macro\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"markers\", \"name\": \"average_precision_score_macro\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.3600624526532358, 0.8644716862755161, 0.6058338786740471, 0.8109487123604503, 0.8747714010147453, 0.7504004261637414, 0.8061792589098291, 0.6232955492224038, 0.8670731273420601, 0.8710563058154909, 0.8372004741698946, 0.5025167509332616, 0.8632478824794572]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"lines\", \"name\": \"average_precision_score_macro_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.3600624526532358, 0.8644716862755161, 0.8644716862755161, 0.8644716862755161, 0.8747714010147453, 0.8747714010147453, 0.8747714010147453, 0.8747714010147453, 0.8747714010147453, 0.8747714010147453, 0.8747714010147453, 0.8747714010147453, 0.8747714010147453]}], \"precision_score_macro\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"markers\", \"name\": \"precision_score_macro\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.3892083459535827, 0.9428958741415487, 0.5278535982151726, 0.8138264617686214, 0.9263510937828079, 0.06014624987609108, 0.6021288990149755, 0.6011779029431631, 0.91814908293542, 0.9175724058787952, 0.8064187773918968, 0.4759725659842525, 0.9091641765283224]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"lines\", \"name\": \"precision_score_macro_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.3892083459535827, 0.9428958741415487, 0.9428958741415487, 0.9428958741415487, 0.9428958741415487, 0.9428958741415487, 0.9428958741415487, 0.9428958741415487, 0.9428958741415487, 0.9428958741415487, 0.9428958741415487, 0.9428958741415487, 0.9428958741415487]}], \"precision_score_micro\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"markers\", \"name\": \"precision_score_micro\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.8831873579244172, 0.9768707267160557, 0.8417425627159593, 0.9766680238942561, 0.977154510666575, 0.8420474982652751, 0.9693342945559897, 0.9061920717932267, 0.9767338288682896, 0.9768583882834244, 0.9707473388644764, 0.9439376873899455, 0.9729835328927925]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"lines\", \"name\": \"precision_score_micro_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.8831873579244172, 0.9768707267160557, 0.9768707267160557, 0.9768707267160557, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575]}], \"precision_score_weighted\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"markers\", \"name\": \"precision_score_weighted\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.9241673808708207, 0.9759630792358593, 0.9149948646584588, 0.9763795549690276, 0.9741048748195125, 0.7090439893348085, 0.9556239292903365, 0.9659204693151312, 0.9712226971662415, 0.9712502477993489, 0.9611770148207437, 0.9247715759845597, 0.9730670628193983]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"lines\", \"name\": \"precision_score_weighted_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.9241673808708207, 0.9759630792358593, 0.9759630792358593, 0.9763795549690276, 0.9763795549690276, 0.9763795549690276, 0.9763795549690276, 0.9763795549690276, 0.9763795549690276, 0.9763795549690276, 0.9763795549690276, 0.9763795549690276, 0.9763795549690276]}], \"norm_macro_recall\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"markers\", \"name\": \"norm_macro_recall\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.4483704606601118, 0.8222790647225984, 0.5348788723208826, 0.7074212453402277, 0.8224897334996466, 0.0, 0.5253895561069603, 0.7859390954409928, 0.8152211822944421, 0.8162991695081769, 0.7015270869650406, 0.37868896918607253, 0.8346544651286354]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"lines\", \"name\": \"norm_macro_recall_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.4483704606601118, 0.8222790647225984, 0.8222790647225984, 0.8222790647225984, 0.8224897334996466, 0.8224897334996466, 0.8224897334996466, 0.8224897334996466, 0.8224897334996466, 0.8224897334996466, 0.8224897334996466, 0.8224897334996466, 0.8346544651286354]}], \"average_precision_score_micro\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"markers\", \"name\": \"average_precision_score_micro\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.7485403993856105, 0.9966166922823638, 0.9371054214240477, 0.992258629783674, 0.997271427334765, 0.9670657135888873, 0.9879370337698163, 0.9399025824240826, 0.9966604294523935, 0.9968854008123046, 0.9922849358805893, 0.9646316991529396, 0.9937617318918045]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"lines\", \"name\": \"average_precision_score_micro_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.7485403993856105, 0.9966166922823638, 0.9966166922823638, 0.9966166922823638, 0.997271427334765, 0.997271427334765, 0.997271427334765, 0.997271427334765, 0.997271427334765, 0.997271427334765, 0.997271427334765, 0.997271427334765, 0.997271427334765]}], \"recall_score_micro\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"markers\", \"name\": \"recall_score_micro\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.8831873579244172, 0.9768707267160557, 0.8417425627159593, 0.9766680238942561, 0.977154510666575, 0.8420474982652751, 0.9693342945559897, 0.9061920717932267, 0.9767338288682896, 0.9768583882834244, 0.9707473388644764, 0.9439376873899455, 0.9729835328927925]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"lines\", \"name\": \"recall_score_micro_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.8831873579244172, 0.9768707267160557, 0.9768707267160557, 0.9768707267160557, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575]}], \"balanced_accuracy\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"markers\", \"name\": \"balanced_accuracy\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.48777257061296103, 0.8349734172424128, 0.5681018100122481, 0.7283197278159257, 0.8351690382496718, 0.07142857142857142, 0.5592903020993202, 0.8012291600523505, 0.8284196692734105, 0.82942065740045, 0.722846580753252, 0.4230683285299245, 0.84646486047659]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"lines\", \"name\": \"balanced_accuracy_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.48777257061296103, 0.8349734172424128, 0.8349734172424128, 0.8349734172424128, 0.8351690382496718, 0.8351690382496718, 0.8351690382496718, 0.8351690382496718, 0.8351690382496718, 0.8351690382496718, 0.8351690382496718, 0.8351690382496718, 0.84646486047659]}], \"recall_score_weighted\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"markers\", \"name\": \"recall_score_weighted\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.8831873579244172, 0.9768707267160557, 0.8417425627159593, 0.9766680238942561, 0.977154510666575, 0.8420474982652751, 0.9693342945559897, 0.9061920717932267, 0.9767338288682896, 0.9768583882834244, 0.9707473388644764, 0.9439376873899455, 0.9729835328927925]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"lines\", \"name\": \"recall_score_weighted_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.8831873579244172, 0.9768707267160557, 0.9768707267160557, 0.9768707267160557, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575]}], \"average_precision_score_weighted\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"markers\", \"name\": \"average_precision_score_weighted\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.9115472505794626, 0.9794886901821938, 0.9612540021050466, 0.976425847282025, 0.9808033097268224, 0.9708483284568964, 0.9738680134422208, 0.9703204236530649, 0.9766559685581083, 0.9771160938374188, 0.967747250711605, 0.9451931051044055, 0.9782523831116801]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"lines\", \"name\": \"average_precision_score_weighted_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.9115472505794626, 0.9794886901821938, 0.9794886901821938, 0.9794886901821938, 0.9808033097268224, 0.9808033097268224, 0.9808033097268224, 0.9808033097268224, 0.9808033097268224, 0.9808033097268224, 0.9808033097268224, 0.9808033097268224, 0.9808033097268224]}], \"f1_score_macro\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"markers\", \"name\": \"f1_score_macro\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.4084802980457898, 0.8509372616733797, 0.4956773349094041, 0.7357756803137994, 0.8533113021033377, 0.06530369052126293, 0.5757982307872712, 0.628506977992991, 0.8457853426514704, 0.8472588930278103, 0.7311938993100658, 0.43591201226808035, 0.8641743907763539]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"lines\", \"name\": \"f1_score_macro_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.4084802980457898, 0.8509372616733797, 0.8509372616733797, 0.8509372616733797, 0.8533113021033377, 0.8533113021033377, 0.8533113021033377, 0.8533113021033377, 0.8533113021033377, 0.8533113021033377, 0.8533113021033377, 0.8533113021033377, 0.8641743907763539]}], \"AUC_macro\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"markers\", \"name\": \"AUC_macro\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.7035942635526415, 0.9902523409691406, 0.9382900442548416, 0.9487795617567515, 0.992468100994752, 0.98232787578062, 0.9847850526254248, 0.9799080394845403, 0.9915045951307121, 0.9931075735718734, 0.9853700233490305, 0.9086130731662473, 0.9867022926602761]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"lines\", \"name\": \"AUC_macro_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.7035942635526415, 0.9902523409691406, 0.9902523409691406, 0.9902523409691406, 0.992468100994752, 0.992468100994752, 0.992468100994752, 0.992468100994752, 0.992468100994752, 0.9931075735718734, 0.9931075735718734, 0.9931075735718734, 0.9931075735718734]}], \"accuracy\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"markers\", \"name\": \"accuracy\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.8831873579244172, 0.9768707267160557, 0.8417425627159593, 0.9766680238942561, 0.977154510666575, 0.8420474982652751, 0.9693342945559897, 0.9061920717932267, 0.9767338288682896, 0.9768583882834244, 0.9707473388644764, 0.9439376873899455, 0.9729835328927925]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"lines\", \"name\": \"accuracy_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.8831873579244172, 0.9768707267160557, 0.9768707267160557, 0.9768707267160557, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575]}], \"f1_score_micro\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"markers\", \"name\": \"f1_score_micro\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.8831873579244172, 0.9768707267160557, 0.8417425627159593, 0.9766680238942561, 0.977154510666575, 0.8420474982652751, 0.9693342945559897, 0.9061920717932267, 0.9767338288682896, 0.9768583882834245, 0.9707473388644764, 0.9439376873899455, 0.9729835328927925]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"lines\", \"name\": \"f1_score_micro_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.8831873579244172, 0.9768707267160557, 0.9768707267160557, 0.9768707267160557, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575, 0.977154510666575]}], \"recall_score_macro\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"markers\", \"name\": \"recall_score_macro\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.48777257061296103, 0.8349734172424128, 0.5681018100122481, 0.7283197278159257, 0.8351690382496718, 0.07142857142857142, 0.5592903020993202, 0.8012291600523505, 0.8284196692734105, 0.82942065740045, 0.722846580753252, 0.4230683285299245, 0.84646486047659]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"lines\", \"name\": \"recall_score_macro_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.48777257061296103, 0.8349734172424128, 0.8349734172424128, 0.8349734172424128, 0.8351690382496718, 0.8351690382496718, 0.8351690382496718, 0.8351690382496718, 0.8351690382496718, 0.8351690382496718, 0.8351690382496718, 0.8351690382496718, 0.84646486047659]}], \"f1_score_weighted\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"markers\", \"name\": \"f1_score_weighted\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.8960278706289995, 0.9706597932002864, 0.875042152733847, 0.9704522472434849, 0.9714764182203561, 0.7698433292328691, 0.9613765416026726, 0.9329946810956897, 0.9704891716532303, 0.9707971758990196, 0.9602830621675655, 0.9336731882535787, 0.9724378086065742]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"lines\", \"name\": \"f1_score_weighted_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.8960278706289995, 0.9706597932002864, 0.9706597932002864, 0.9706597932002864, 0.9714764182203561, 0.9714764182203561, 0.9714764182203561, 0.9714764182203561, 0.9714764182203561, 0.9714764182203561, 0.9714764182203561, 0.9714764182203561, 0.9724378086065742]}], \"AUC_weighted\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"markers\", \"name\": \"AUC_weighted\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.900650609951382, 0.9925684683486623, 0.9713515875602939, 0.9840793169378296, 0.9945318913276524, 0.9861746099818539, 0.9905217292374083, 0.9796774722469993, 0.9932548262112253, 0.9939722736117674, 0.9866355229100155, 0.938571892637819, 0.9882813889438005]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"lines\", \"name\": \"AUC_weighted_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.900650609951382, 0.9925684683486623, 0.9925684683486623, 0.9925684683486623, 0.9945318913276524, 0.9945318913276524, 0.9945318913276524, 0.9945318913276524, 0.9945318913276524, 0.9945318913276524, 0.9945318913276524, 0.9945318913276524, 0.9945318913276524]}], \"weighted_accuracy\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"markers\", \"name\": \"weighted_accuracy\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.9028617219744152, 0.9994608982393822, 0.9005737488968931, 0.9994621071703221, 0.9992676804176687, 0.9949766796074421, 0.9993056317453022, 0.9120807810534683, 0.9993383990664866, 0.9992654401113183, 0.9989426340920614, 0.988626001871342, 0.9902775001227279]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"lines\", \"name\": \"weighted_accuracy_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.9028617219744152, 0.9994608982393822, 0.9994608982393822, 0.9994621071703221, 0.9994621071703221, 0.9994621071703221, 0.9994621071703221, 0.9994621071703221, 0.9994621071703221, 0.9994621071703221, 0.9994621071703221, 0.9994621071703221, 0.9994621071703221]}], \"AUC_micro\": [{\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"markers\", \"name\": \"AUC_micro\", \"stepped\": false, \"type\": \"scatter\", \"data\": [0.9213933766576152, 0.999744856440352, 0.9901378289727318, 0.9993416626167417, 0.999792960795072, 0.9944917000837896, 0.9980862564081511, 0.9946523027317417, 0.9997495371919567, 0.9997672321563977, 0.9994479524235571, 0.9966335692800982, 0.9995343721531104]}, {\"categories\": [\"0\", \"1\", \"2\", \"3\", \"4\", \"6\", \"7\", \"8\", \"9\", \"10\", \"12\", \"13\", \"14\"], \"mode\": \"lines\", \"name\": \"AUC_micro_max\", \"stepped\": true, \"type\": \"scatter\", \"data\": [0.9213933766576152, 0.999744856440352, 0.999744856440352, 0.999744856440352, 0.999792960795072, 0.999792960795072, 0.999792960795072, 0.999792960795072, 0.999792960795072, 0.999792960795072, 0.999792960795072, 0.999792960795072, 0.999792960795072]}]}, \"metricName\": null, \"primaryMetricName\": \"norm_macro_recall\", \"showLegend\": false}, \"run_metrics\": [{\"name\": \"experiment_status\", \"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"categories\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], \"series\": [{\"data\": [\"DatasetEvaluation\", \"FeaturesGeneration\", \"DatasetFeaturization\", \"DatasetFeaturizationCompleted\", \"DatasetBalancing\", \"ModelSelection\", \"BestRunExplainModel\", \"ModelExplanationDataSetSetup\", \"PickSurrogateModel\", \"EngineeredFeatureExplanations\", \"EngineeredFeatureExplanations\", \"RawFeaturesExplanations\", \"RawFeaturesExplanations\", \"BestRunExplainModel\"]}]}, {\"name\": \"experiment_status_description\", \"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"categories\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], \"series\": [{\"data\": [\"Gathering dataset statistics.\", \"Generating features for the dataset.\", \"Beginning to fit featurizers and featurize the dataset.\", \"Completed fit featurizers and featurizing the dataset.\", \"Performing class balancing sweeping\", \"Beginning model selection.\", \"Best run model explanations started\", \"Model explanations data setup completed\", \"Choosing LightGBM as the surrogate model for explanations\", \"Computation of engineered features started\", \"Computation of engineered features completed\", \"Computation of raw features started\", \"Computation of raw features completed\", \"Best run model explanations completed\"]}]}, {\"name\": \"weighted_accuracy\", \"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"categories\": [0], \"series\": [{\"data\": [0.9902775001227279]}]}, {\"name\": \"recall_score_micro\", \"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"categories\": [0], \"series\": [{\"data\": [0.9729835328927925]}]}, {\"name\": \"AUC_micro\", \"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"categories\": [0], \"series\": [{\"data\": [0.9995343721531104]}]}, {\"name\": \"accuracy\", \"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"categories\": [0], \"series\": [{\"data\": [0.9729835328927925]}]}, {\"name\": \"AUC_weighted\", \"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"categories\": [0], \"series\": [{\"data\": [0.9882813889438005]}]}, {\"name\": \"f1_score_micro\", \"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"categories\": [0], \"series\": [{\"data\": [0.9729835328927925]}]}, {\"name\": \"average_precision_score_weighted\", \"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"categories\": [0], \"series\": [{\"data\": [0.9782523831116801]}]}, {\"name\": \"AUC_macro\", \"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"categories\": [0], \"series\": [{\"data\": [0.9867022926602761]}]}, {\"name\": \"recall_score_weighted\", \"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"categories\": [0], \"series\": [{\"data\": [0.9729835328927925]}]}, {\"name\": \"matthews_correlation\", \"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"categories\": [0], \"series\": [{\"data\": [0.9060454227132824]}]}, {\"name\": \"f1_score_weighted\", \"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"categories\": [0], \"series\": [{\"data\": [0.9724378086065742]}]}, {\"name\": \"norm_macro_recall\", \"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"categories\": [0], \"series\": [{\"data\": [0.8346544651286354]}]}, {\"name\": \"recall_score_macro\", \"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"categories\": [0], \"series\": [{\"data\": [0.84646486047659]}]}, {\"name\": \"precision_score_weighted\", \"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"categories\": [0], \"series\": [{\"data\": [0.9730670628193983]}]}, {\"name\": \"balanced_accuracy\", \"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"categories\": [0], \"series\": [{\"data\": [0.84646486047659]}]}, {\"name\": \"precision_score_macro\", \"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"categories\": [0], \"series\": [{\"data\": [0.9091641765283224]}]}, {\"name\": \"f1_score_macro\", \"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"categories\": [0], \"series\": [{\"data\": [0.8641743907763539]}]}, {\"name\": \"log_loss\", \"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"categories\": [0], \"series\": [{\"data\": [0.22456691217636446]}]}, {\"name\": \"precision_score_micro\", \"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"categories\": [0], \"series\": [{\"data\": [0.9729835328927925]}]}, {\"name\": \"average_precision_score_macro\", \"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"categories\": [0], \"series\": [{\"data\": [0.8632478824794572]}]}, {\"name\": \"average_precision_score_micro\", \"run_id\": \"AutoML_ae1d998c-ec87-447d-bf70-328476f80063\", \"categories\": [0], \"series\": [{\"data\": [0.9937617318918045]}]}], \"run_logs\": \"\\nRun is completed.\", \"graph\": {}, \"widget_settings\": {\"childWidgetDisplay\": \"popup\", \"send_telemetry\": false, \"log_level\": \"INFO\", \"sdk_version\": \"1.21.0\"}, \"loading\": false}"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "RunDetails(automl_run).show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Best Model\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Retrieve the best run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "gather": {
     "logged": 1598546650307
    },
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    }
   },
   "outputs": [],
   "source": [
    "best_run = package_utils.get_best_run(experiment, automl_run)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run(Experiment: jupyter-2021-01-29-09-02,\n",
      "Id: AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14,\n",
      "Type: azureml.scriptrun,\n",
      "Status: Completed)\n",
      "{'average_precision_score_macro': 0.8632478824794572, 'precision_score_micro': 0.9729835328927925, 'f1_score_weighted': 0.9724378086065742, 'weighted_accuracy': 0.9902775001227279, 'f1_score_micro': 0.9729835328927925, 'norm_macro_recall': 0.8346544651286354, 'average_precision_score_micro': 0.9937617318918045, 'AUC_macro': 0.9867022926602761, 'AUC_micro': 0.9995343721531104, 'precision_score_weighted': 0.9730670628193983, 'AUC_weighted': 0.9882813889438005, 'f1_score_macro': 0.8641743907763539, 'recall_score_macro': 0.84646486047659, 'log_loss': 0.22456691217636446, 'accuracy': 0.9729835328927925, 'recall_score_micro': 0.9729835328927925, 'recall_score_weighted': 0.9729835328927925, 'matthews_correlation': 0.9060454227132824, 'balanced_accuracy': 0.84646486047659, 'precision_score_macro': 0.9091641765283224, 'average_precision_score_weighted': 0.9782523831116801, 'accuracy_table': 'aml://artifactId/ExperimentRun/dcid.AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14/accuracy_table', 'confusion_matrix': 'aml://artifactId/ExperimentRun/dcid.AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14/confusion_matrix'}\n"
     ]
    }
   ],
   "source": [
    "print(best_run)\n",
    "print(best_run.get_metrics())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save the best model locally"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'runId': 'AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14', 'target': 'starts-ws-136633', 'status': 'Completed', 'startTimeUtc': '2021-01-30T20:49:34.62987Z', 'endTimeUtc': '2021-01-30T21:18:11.219349Z', 'properties': {'runTemplate': 'automl_child', 'pipeline_id': '__AutoML_Ensemble__', 'pipeline_spec': '{\"pipeline_id\":\"__AutoML_Ensemble__\",\"objects\":[{\"module\":\"azureml.train.automl.ensemble\",\"class_name\":\"Ensemble\",\"spec_class\":\"sklearn\",\"param_args\":[],\"param_kwargs\":{\"automl_settings\":\"{\\'task_type\\':\\'classification\\',\\'primary_metric\\':\\'norm_macro_recall\\',\\'verbosity\\':20,\\'ensemble_iterations\\':15,\\'is_timeseries\\':False,\\'name\\':\\'jupyter-2021-01-29-09-02\\',\\'compute_target\\':\\'starts-ws-136633\\',\\'subscription_id\\':\\'a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\\',\\'region\\':\\'southcentralus\\',\\'spark_service\\':None}\",\"ensemble_run_id\":\"AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14\",\"experiment_name\":\"jupyter-2021-01-29-09-02\",\"workspace_name\":\"quick-starts-ws-136633\",\"subscription_id\":\"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\",\"resource_group_name\":\"aml-quickstarts-136633\"}}]}', 'training_percent': '100', 'predicted_cost': None, 'iteration': '14', '_aml_system_scenario_identification': 'Remote.Child', '_azureml.ComputeTargetType': 'amlcompute', 'ContentSnapshotId': 'aaf2d83f-96e4-42e6-bc35-ee3257c53b1b', 'ProcessInfoFile': 'azureml-logs/process_info.json', 'ProcessStatusFile': 'azureml-logs/process_status.json', 'run_template': 'automl_child', 'run_preprocessor': '', 'run_algorithm': 'VotingEnsemble', 'conda_env_data_location': 'aml://artifact/ExperimentRun/dcid.AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14/outputs/conda_env_v_1_0_0.yml', 'model_data_location': 'aml://artifact/ExperimentRun/dcid.AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14/outputs/model.pkl', 'model_size_on_disk': '13767058', 'scoring_data_location': 'aml://artifact/ExperimentRun/dcid.AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14/outputs/scoring_file_v_1_0_0.py', 'model_exp_support': 'True', 'pipeline_graph_version': '1.0.0', 'model_name': 'AutoMLae1d998ce14', 'staticProperties': '{}', 'score': '0.8346544651286354', 'run_properties': \"classification_labels=None,\\n                              estimators=[('4',\\n                                           Pipeline(memory=None,\\n                                                    steps=[('standardscalerwrapper',\\n                                                            <azureml.automl.runtime.shared.model_wrappers.StandardScalerWrapper object at 0x7f40840bd550>\", 'pipeline_script': '{\"pipeline_id\":\"__AutoML_Ensemble__\",\"objects\":[{\"module\":\"azureml.train.automl.ensemble\",\"class_name\":\"Ensemble\",\"spec_class\":\"sklearn\",\"param_args\":[],\"param_kwargs\":{\"automl_settings\":\"{\\'task_type\\':\\'classification\\',\\'primary_metric\\':\\'norm_macro_recall\\',\\'verbosity\\':20,\\'ensemble_iterations\\':15,\\'is_timeseries\\':False,\\'name\\':\\'jupyter-2021-01-29-09-02\\',\\'compute_target\\':\\'starts-ws-136633\\',\\'subscription_id\\':\\'a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\\',\\'region\\':\\'southcentralus\\',\\'spark_service\\':None}\",\"ensemble_run_id\":\"AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14\",\"experiment_name\":\"jupyter-2021-01-29-09-02\",\"workspace_name\":\"quick-starts-ws-136633\",\"subscription_id\":\"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\",\"resource_group_name\":\"aml-quickstarts-136633\"}}]}', 'training_type': 'train_valid', 'num_classes': '', 'framework': 'sklearn', 'fit_time': '1167', 'goal': 'norm_macro_recall_max', 'class_labels': '', 'primary_metric': 'norm_macro_recall', 'errors': '{}', 'onnx_model_resource': '{}', 'error_code': '', 'failure_reason': '', 'feature_skus': 'automatedml_sdk_guardrails', 'dependencies_versions': '{\"azureml-train-automl-runtime\": \"1.20.0.post1\", \"azureml-train-automl-client\": \"1.20.0\", \"azureml-telemetry\": \"1.20.0\", \"azureml-pipeline-core\": \"1.20.0\", \"azureml-model-management-sdk\": \"1.0.1b6.post1\", \"azureml-interpret\": \"1.20.0\", \"azureml-defaults\": \"1.20.0\", \"azureml-dataset-runtime\": \"1.20.0\", \"azureml-dataprep\": \"2.7.3\", \"azureml-dataprep-rslex\": \"1.5.0\", \"azureml-dataprep-native\": \"27.0.0\", \"azureml-core\": \"1.20.0\", \"azureml-automl-runtime\": \"1.20.0\", \"azureml-automl-core\": \"1.20.0\"}', 'num_cores': '4', 'num_logical_cores': '4', 'peak_memory_usage': '8015464', 'vm_configuration': 'Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz', 'core_hours': '0.9499898530555556'}, 'inputDatasets': [], 'outputDatasets': [], 'runDefinition': {'script': 'automl_driver.py', 'command': '', 'useAbsolutePath': False, 'arguments': [], 'sourceDirectoryDataStore': None, 'framework': 'Python', 'communicator': 'None', 'target': 'starts-ws-136633', 'dataReferences': {}, 'data': {}, 'outputData': {}, 'jobName': None, 'maxRunDurationSeconds': None, 'nodeCount': 1, 'priority': None, 'credentialPassthrough': False, 'environment': {'name': 'AutoML-AzureML-AutoML', 'version': 'Autosave_2021-01-30T19:03:43Z_a2054c1b', 'python': {'interpreterPath': 'python', 'userManagedDependencies': False, 'condaDependencies': {'channels': ['anaconda', 'conda-forge', 'pytorch'], 'dependencies': ['python=3.6.2', 'pip=20.2.4', {'pip': ['azureml-core==1.20.0', 'azureml-pipeline-core==1.20.0', 'azureml-telemetry==1.20.0', 'azureml-defaults==1.20.0', 'azureml-interpret==1.20.0', 'azureml-automl-core==1.20.0', 'azureml-automl-runtime==1.20.0', 'azureml-train-automl-client==1.20.0', 'azureml-train-automl-runtime==1.20.0.post1', 'azureml-dataset-runtime==1.20.0', 'inference-schema', 'py-cpuinfo==5.0.0', 'boto3==1.15.18', 'botocore==1.18.18']}, 'numpy~=1.18.0', 'scikit-learn==0.22.1', 'pandas~=0.25.0', 'py-xgboost<=0.90', 'fbprophet==0.5', 'holidays==0.9.11', 'setuptools-git', 'psutil>5.0.0,<6.0.0'], 'name': 'azureml_265db83b0c6014ce472c5de2f0b97e04'}, 'baseCondaEnvironment': None}, 'environmentVariables': {'EXAMPLE_ENV_VAR': 'EXAMPLE_VALUE'}, 'docker': {'baseImage': 'mcr.microsoft.com/azureml/intelmpi2018.3-ubuntu16.04:20210104.v1', 'platform': {'os': 'Linux', 'architecture': 'amd64'}, 'baseDockerfile': None, 'baseImageRegistry': {'address': None, 'username': None, 'password': None}, 'enabled': True, 'arguments': []}, 'spark': {'repositories': [], 'packages': [], 'precachePackages': True}, 'inferencingStackVersion': None}, 'history': {'outputCollection': True, 'directoriesToWatch': ['logs'], 'enableMLflowTracking': True}, 'spark': {'configuration': {'spark.app.name': 'Azure ML Experiment', 'spark.yarn.maxAppAttempts': '1'}}, 'parallelTask': {'maxRetriesPerWorker': 0, 'workerCountPerNode': 1, 'terminalExitCodes': None, 'configuration': {}}, 'amlCompute': {'name': None, 'vmSize': None, 'retainCluster': False, 'clusterMaxNodeCount': None}, 'aiSuperComputer': {'instanceType': None, 'frameworkImage': None, 'imageVersion': None, 'location': None, 'aiSuperComputerStorageData': None, 'interactive': False, 'scalePolicy': None}, 'tensorflow': {'workerCount': 1, 'parameterServerCount': 1}, 'mpi': {'processCountPerNode': 1}, 'pyTorch': {'communicationBackend': None, 'processCount': None}, 'hdi': {'yarnDeployMode': 'Cluster'}, 'containerInstance': {'region': None, 'cpuCores': 2.0, 'memoryGb': 3.5}, 'exposedPorts': None, 'docker': {'useDocker': True, 'sharedVolumes': True, 'shmSize': '2g', 'arguments': []}, 'cmk8sCompute': {'configuration': {}}, 'commandReturnCodeConfig': {'returnCode': 'Zero', 'successfulReturnCodes': []}}, 'logFiles': {'azureml-logs/55_azureml-execution-tvmps_fb457a87b5a3e48452dcc5a2beafe8fe1e6cd84aff0c08302b4299be59eed3c5_p.txt': 'https://mlstrg136633.blob.core.windows.net/azureml/ExperimentRun/dcid.AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14/azureml-logs/55_azureml-execution-tvmps_fb457a87b5a3e48452dcc5a2beafe8fe1e6cd84aff0c08302b4299be59eed3c5_p.txt?sv=2019-02-02&sr=b&sig=qrZOS8bVpjf4q6g3AfM6OpGLa9mKUMMQx%2FRD3w96wok%3D&st=2021-01-30T21%3A09%3A53Z&se=2021-01-31T05%3A19%3A53Z&sp=r', 'azureml-logs/65_job_prep-tvmps_fb457a87b5a3e48452dcc5a2beafe8fe1e6cd84aff0c08302b4299be59eed3c5_p.txt': 'https://mlstrg136633.blob.core.windows.net/azureml/ExperimentRun/dcid.AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14/azureml-logs/65_job_prep-tvmps_fb457a87b5a3e48452dcc5a2beafe8fe1e6cd84aff0c08302b4299be59eed3c5_p.txt?sv=2019-02-02&sr=b&sig=54vSg%2FAovuoglyqaBXf3g0g53%2FZxqdodgJ5rBr9YXdA%3D&st=2021-01-30T21%3A09%3A53Z&se=2021-01-31T05%3A19%3A53Z&sp=r', 'azureml-logs/70_driver_log.txt': 'https://mlstrg136633.blob.core.windows.net/azureml/ExperimentRun/dcid.AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14/azureml-logs/70_driver_log.txt?sv=2019-02-02&sr=b&sig=wXWfbmDQSF0KzL4w9QFUXS%2BzF2LykNt50v3yEassvPU%3D&st=2021-01-30T21%3A09%3A53Z&se=2021-01-31T05%3A19%3A53Z&sp=r', 'azureml-logs/75_job_post-tvmps_fb457a87b5a3e48452dcc5a2beafe8fe1e6cd84aff0c08302b4299be59eed3c5_p.txt': 'https://mlstrg136633.blob.core.windows.net/azureml/ExperimentRun/dcid.AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14/azureml-logs/75_job_post-tvmps_fb457a87b5a3e48452dcc5a2beafe8fe1e6cd84aff0c08302b4299be59eed3c5_p.txt?sv=2019-02-02&sr=b&sig=NyNS4CFtos4QWovaQCRbaWWrLNjWUZL6sDnJYkvtBto%3D&st=2021-01-30T21%3A09%3A53Z&se=2021-01-31T05%3A19%3A53Z&sp=r', 'azureml-logs/process_info.json': 'https://mlstrg136633.blob.core.windows.net/azureml/ExperimentRun/dcid.AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14/azureml-logs/process_info.json?sv=2019-02-02&sr=b&sig=ZM1IMLxxyrSgkkpDKcl9Qdhss0K9c%2FPDLxbBKWu0bYc%3D&st=2021-01-30T21%3A09%3A53Z&se=2021-01-31T05%3A19%3A53Z&sp=r', 'azureml-logs/process_status.json': 'https://mlstrg136633.blob.core.windows.net/azureml/ExperimentRun/dcid.AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14/azureml-logs/process_status.json?sv=2019-02-02&sr=b&sig=DtP%2FDZm%2Bn9nT4TNjPLoTDptJG5wQT%2BjFCF%2Fk0iHDZs8%3D&st=2021-01-30T21%3A09%3A53Z&se=2021-01-31T05%3A19%3A53Z&sp=r', 'logs/azureml/108_azureml.log': 'https://mlstrg136633.blob.core.windows.net/azureml/ExperimentRun/dcid.AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14/logs/azureml/108_azureml.log?sv=2019-02-02&sr=b&sig=hDhIFGGR2ExbgSLbzhwgqT%2FsE1ystAXfKrZjUAMA3EM%3D&st=2021-01-30T21%3A09%3A53Z&se=2021-01-31T05%3A19%3A53Z&sp=r', 'logs/azureml/azureml_automl.log': 'https://mlstrg136633.blob.core.windows.net/azureml/ExperimentRun/dcid.AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14/logs/azureml/azureml_automl.log?sv=2019-02-02&sr=b&sig=f6NwpcePmtX0XbcmzkAdPAW6ezCfSgN87LGnU9cHNW4%3D&st=2021-01-30T21%3A09%3A53Z&se=2021-01-31T05%3A19%3A53Z&sp=r', 'logs/azureml/dataprep/python_span_51f4d14a-00da-4251-866c-5c0cb116123c.jsonl': 'https://mlstrg136633.blob.core.windows.net/azureml/ExperimentRun/dcid.AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14/logs/azureml/dataprep/python_span_51f4d14a-00da-4251-866c-5c0cb116123c.jsonl?sv=2019-02-02&sr=b&sig=fN9YIHdgH6z1R20kE3a4gYjN2dMxRceaatepY28UWs8%3D&st=2021-01-30T21%3A09%3A53Z&se=2021-01-31T05%3A19%3A53Z&sp=r', 'logs/azureml/dataprep/python_span_63a54182-da6e-47db-8ab6-6038f8923ee9.jsonl': 'https://mlstrg136633.blob.core.windows.net/azureml/ExperimentRun/dcid.AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14/logs/azureml/dataprep/python_span_63a54182-da6e-47db-8ab6-6038f8923ee9.jsonl?sv=2019-02-02&sr=b&sig=TtfjbnrCtAU%2FUl00zExEpTHW%2FlRLIE6L5aF%2FEf%2FRMok%3D&st=2021-01-30T21%3A09%3A53Z&se=2021-01-31T05%3A19%3A53Z&sp=r', 'logs/azureml/job_prep_azureml.log': 'https://mlstrg136633.blob.core.windows.net/azureml/ExperimentRun/dcid.AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14/logs/azureml/job_prep_azureml.log?sv=2019-02-02&sr=b&sig=iuurBeQIQ5%2B4jq8V9x9BX%2FZmaNxBcBf1QCVi91gmjO0%3D&st=2021-01-30T21%3A09%3A53Z&se=2021-01-31T05%3A19%3A53Z&sp=r', 'logs/azureml/job_release_azureml.log': 'https://mlstrg136633.blob.core.windows.net/azureml/ExperimentRun/dcid.AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14/logs/azureml/job_release_azureml.log?sv=2019-02-02&sr=b&sig=NR5wddMXmQiPnK4oLFqrZIyF9qwXyOQMdUjSeOVcet8%3D&st=2021-01-30T21%3A09%3A53Z&se=2021-01-31T05%3A19%3A53Z&sp=r'}, 'submittedBy': 'ODL_User 136633'}\n"
     ]
    }
   ],
   "source": [
    "print(best_run.get_details())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'runTemplate': 'automl_child', 'pipeline_id': '__AutoML_Ensemble__', 'pipeline_spec': '{\"pipeline_id\":\"__AutoML_Ensemble__\",\"objects\":[{\"module\":\"azureml.train.automl.ensemble\",\"class_name\":\"Ensemble\",\"spec_class\":\"sklearn\",\"param_args\":[],\"param_kwargs\":{\"automl_settings\":\"{\\'task_type\\':\\'classification\\',\\'primary_metric\\':\\'norm_macro_recall\\',\\'verbosity\\':20,\\'ensemble_iterations\\':15,\\'is_timeseries\\':False,\\'name\\':\\'jupyter-2021-01-29-09-02\\',\\'compute_target\\':\\'starts-ws-136633\\',\\'subscription_id\\':\\'a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\\',\\'region\\':\\'southcentralus\\',\\'spark_service\\':None}\",\"ensemble_run_id\":\"AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14\",\"experiment_name\":\"jupyter-2021-01-29-09-02\",\"workspace_name\":\"quick-starts-ws-136633\",\"subscription_id\":\"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\",\"resource_group_name\":\"aml-quickstarts-136633\"}}]}', 'training_percent': '100', 'predicted_cost': None, 'iteration': '14', '_aml_system_scenario_identification': 'Remote.Child', '_azureml.ComputeTargetType': 'amlcompute', 'ContentSnapshotId': 'aaf2d83f-96e4-42e6-bc35-ee3257c53b1b', 'ProcessInfoFile': 'azureml-logs/process_info.json', 'ProcessStatusFile': 'azureml-logs/process_status.json', 'run_template': 'automl_child', 'run_preprocessor': '', 'run_algorithm': 'VotingEnsemble', 'conda_env_data_location': 'aml://artifact/ExperimentRun/dcid.AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14/outputs/conda_env_v_1_0_0.yml', 'model_data_location': 'aml://artifact/ExperimentRun/dcid.AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14/outputs/model.pkl', 'model_size_on_disk': '13767058', 'scoring_data_location': 'aml://artifact/ExperimentRun/dcid.AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14/outputs/scoring_file_v_1_0_0.py', 'model_exp_support': 'True', 'pipeline_graph_version': '1.0.0', 'model_name': 'AutoMLae1d998ce14', 'staticProperties': '{}', 'score': '0.8346544651286354', 'run_properties': \"classification_labels=None,\\n                              estimators=[('4',\\n                                           Pipeline(memory=None,\\n                                                    steps=[('standardscalerwrapper',\\n                                                            <azureml.automl.runtime.shared.model_wrappers.StandardScalerWrapper object at 0x7f40840bd550>\", 'pipeline_script': '{\"pipeline_id\":\"__AutoML_Ensemble__\",\"objects\":[{\"module\":\"azureml.train.automl.ensemble\",\"class_name\":\"Ensemble\",\"spec_class\":\"sklearn\",\"param_args\":[],\"param_kwargs\":{\"automl_settings\":\"{\\'task_type\\':\\'classification\\',\\'primary_metric\\':\\'norm_macro_recall\\',\\'verbosity\\':20,\\'ensemble_iterations\\':15,\\'is_timeseries\\':False,\\'name\\':\\'jupyter-2021-01-29-09-02\\',\\'compute_target\\':\\'starts-ws-136633\\',\\'subscription_id\\':\\'a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\\',\\'region\\':\\'southcentralus\\',\\'spark_service\\':None}\",\"ensemble_run_id\":\"AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14\",\"experiment_name\":\"jupyter-2021-01-29-09-02\",\"workspace_name\":\"quick-starts-ws-136633\",\"subscription_id\":\"a24a24d5-8d87-4c8a-99b6-91ed2d2df51f\",\"resource_group_name\":\"aml-quickstarts-136633\"}}]}', 'training_type': 'train_valid', 'num_classes': '', 'framework': 'sklearn', 'fit_time': '1167', 'goal': 'norm_macro_recall_max', 'class_labels': '', 'primary_metric': 'norm_macro_recall', 'errors': '{}', 'onnx_model_resource': '{}', 'error_code': '', 'failure_reason': '', 'feature_skus': 'automatedml_sdk_guardrails', 'dependencies_versions': '{\"azureml-train-automl-runtime\": \"1.20.0.post1\", \"azureml-train-automl-client\": \"1.20.0\", \"azureml-telemetry\": \"1.20.0\", \"azureml-pipeline-core\": \"1.20.0\", \"azureml-model-management-sdk\": \"1.0.1b6.post1\", \"azureml-interpret\": \"1.20.0\", \"azureml-defaults\": \"1.20.0\", \"azureml-dataset-runtime\": \"1.20.0\", \"azureml-dataprep\": \"2.7.3\", \"azureml-dataprep-rslex\": \"1.5.0\", \"azureml-dataprep-native\": \"27.0.0\", \"azureml-core\": \"1.20.0\", \"azureml-automl-runtime\": \"1.20.0\", \"azureml-automl-core\": \"1.20.0\"}', 'num_cores': '4', 'num_logical_cores': '4', 'peak_memory_usage': '8015464', 'vm_configuration': 'Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz', 'core_hours': '0.9499898530555556'}\n"
     ]
    }
   ],
   "source": [
    "print(best_run.get_properties())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.35 s, sys: 719 ms, total: 2.07 s\n",
      "Wall time: 28.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "best_run.download_file(\"outputs/model.pkl\", \"models/automl/model.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model size in Bytes 13767058\n"
     ]
    }
   ],
   "source": [
    "print(\"model size in Bytes\", pathlib.Path(\"models/automl/model.pkl\").stat().st_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Download other files from the best_run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['accuracy_table', 'automl_driver.py', 'azureml-logs/55_azureml-execution-tvmps_fb457a87b5a3e48452dcc5a2beafe8fe1e6cd84aff0c08302b4299be59eed3c5_p.txt', 'azureml-logs/65_job_prep-tvmps_fb457a87b5a3e48452dcc5a2beafe8fe1e6cd84aff0c08302b4299be59eed3c5_p.txt', 'azureml-logs/70_driver_log.txt', 'azureml-logs/75_job_post-tvmps_fb457a87b5a3e48452dcc5a2beafe8fe1e6cd84aff0c08302b4299be59eed3c5_p.txt', 'azureml-logs/process_info.json', 'azureml-logs/process_status.json', 'confusion_matrix', 'logs/azureml/108_azureml.log', 'logs/azureml/azureml_automl.log', 'logs/azureml/dataprep/python_span_51f4d14a-00da-4251-866c-5c0cb116123c.jsonl', 'logs/azureml/dataprep/python_span_63a54182-da6e-47db-8ab6-6038f8923ee9.jsonl', 'logs/azureml/job_prep_azureml.log', 'logs/azureml/job_release_azureml.log', 'outputs/conda_env_v_1_0_0.yml', 'outputs/env_dependencies.json', 'outputs/model.pkl', 'outputs/pipeline_graph.json', 'outputs/scoring_file_v_1_0_0.py']\n"
     ]
    }
   ],
   "source": [
    "print(best_run.get_file_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_run.download_file(\"outputs/conda_env_v_1_0_0.yml\",\n",
    "                       \"models/automl/conda_env_v_1_0_0.yml\")\n",
    "best_run.download_file(\"outputs/scoring_file_v_1_0_0.py\",\n",
    "                       \"models/automl/scoring_file_v_1_0_0.py\")\n",
    "best_run.download_file(\"outputs/env_dependencies.json\",\n",
    "                       \"models/automl/env_dependencies.json\")\n",
    "\n",
    "best_run.download_file(\"automl_driver.py\",\n",
    "                       \"models/automl/automl_driver.py\")\n",
    "best_run.download_file(\"logs/azureml/azureml_automl.log\",\n",
    "                       \"models/automl/azureml_automl.log\")\n",
    "best_run.download_file(\"azureml-logs/70_driver_log.txt\",\n",
    "                       \"models/automl/70_driver_log.txt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Explore the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 10 µs, sys: 2 µs, total: 12 µs\n",
      "Wall time: 21.7 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "if 0:  # This method works only for AutoML\n",
    "    _, fitted_model = automl_run.get_output()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.01 s, sys: 336 ms, total: 2.35 s\n",
      "Wall time: 2.61 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "fitted_model = joblib.load(\"models/automl/model.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'prefittedsoftvotingclassifier__estimators': [XGBoostClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
      "                  colsample_bynode=1, colsample_bytree=1, eta=0.001, gamma=0,\n",
      "                  learning_rate=0.1, max_delta_step=0, max_depth=10,\n",
      "                  max_leaves=15, min_child_weight=1, missing=nan,\n",
      "                  n_estimators=100, n_jobs=-1, nthread=None,\n",
      "                  objective='multi:softprob', random_state=0,\n",
      "                  reg_alpha=2.0833333333333335, reg_lambda=0.9375,\n",
      "                  scale_pos_weight=1, seed=None, silent=None, subsample=0.9,\n",
      "                  tree_method='auto', verbose=-10, verbosity=0),\n",
      "                                               XGBoostClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
      "                  colsample_bynode=1, colsample_bytree=1, gamma=0,\n",
      "                  learning_rate=0.1, max_delta_step=0, max_depth=3,\n",
      "                  min_child_weight=1, missing=nan, n_estimators=100, n_jobs=-1,\n",
      "                  nthread=None, objective='multi:softprob', random_state=0,\n",
      "                  reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
      "                  silent=None, subsample=1, tree_method='auto', verbose=-10,\n",
      "                  verbosity=0),\n",
      "                                               LightGBMClassifier(boosting_type='goss', class_weight=None,\n",
      "                   colsample_bytree=0.8911111111111111, importance_type='split',\n",
      "                   learning_rate=0.08947473684210526, max_bin=200, max_depth=10,\n",
      "                   min_child_samples=41119, min_child_weight=7,\n",
      "                   min_split_gain=0.42105263157894735, n_estimators=800,\n",
      "                   n_jobs=-1, num_leaves=119, objective=None, random_state=None,\n",
      "                   reg_alpha=0.7368421052631579, reg_lambda=0.2631578947368421,\n",
      "                   silent=True, subsample=1, subsample_for_bin=200000,\n",
      "                   subsample_freq=0, verbose=-10),\n",
      "                                               LightGBMClassifier(boosting_type='goss', class_weight=None,\n",
      "                   colsample_bytree=0.8911111111111111, importance_type='split',\n",
      "                   learning_rate=0.1, max_bin=180, max_depth=10,\n",
      "                   min_child_samples=54812, min_child_weight=9,\n",
      "                   min_split_gain=0.15789473684210525, n_estimators=200,\n",
      "                   n_jobs=-1, num_leaves=53, objective=None, random_state=None,\n",
      "                   reg_alpha=0.42105263157894735,\n",
      "                   reg_lambda=0.15789473684210525, silent=True, subsample=1,\n",
      "                   subsample_for_bin=200000, subsample_freq=0, verbose=-10),\n",
      "                                               LogisticRegression(C=0.8286427728546842, class_weight='balanced', dual=False,\n",
      "                   fit_intercept=True, intercept_scaling=1, l1_ratio=None,\n",
      "                   max_iter=100, multi_class='ovr', n_jobs=-1, penalty='l2',\n",
      "                   random_state=None, solver='newton-cg', tol=0.0001, verbose=0,\n",
      "                   warm_start=False),\n",
      "                                               SGDClassifierWrapper(alpha=1.6327367346938775, class_weight='balanced',\n",
      "                     eta0=0.001, fit_intercept=True,\n",
      "                     l1_ratio=0.8571428571428571, learning_rate='constant',\n",
      "                     loss='modified_huber', max_iter=1000, n_jobs=-1,\n",
      "                     penalty='none', power_t=0.7777777777777777,\n",
      "                     random_state=None, tol=0.0001)],\n",
      " 'prefittedsoftvotingclassifier__weights': [0.14285714285714285,\n",
      "                                            0.07142857142857142,\n",
      "                                            0.14285714285714285,\n",
      "                                            0.07142857142857142,\n",
      "                                            0.42857142857142855,\n",
      "                                            0.14285714285714285]}\n"
     ]
    }
   ],
   "source": [
    "info = {}\n",
    "\n",
    "for key, value in fitted_model.get_params()[\"Pipeline\"].items():\n",
    "    if key == \"prefittedsoftvotingclassifier__weights\":\n",
    "        info[key] = value\n",
    "    if key == \"prefittedsoftvotingclassifier__estimators\":\n",
    "        info[key] = []\n",
    "        for estimator in value:\n",
    "            constitute_model = estimator[1].steps[-1][-1]\n",
    "            info[key].append(constitute_model)\n",
    "\n",
    "pprint.pprint(info)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model testing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test the model on the test set from 2018 and on an additional out-of-sample test set from 2017."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test the model on the 2018 dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 23.5 s, sys: 17.5 s, total: 41 s\n",
      "Wall time: 2min 13s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "test = Dataset.get_by_name(\n",
    "            workspace=workspace,\n",
    "            name=dataset_test_name,\n",
    "        ).to_pandas_dataframe()\n",
    "x_test, y_test = test.drop(labels=[\"Label\"], axis=1), test[\"Label\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1h 38min 33s, sys: 6min 16s, total: 1h 44min 49s\n",
      "Wall time: 42min 50s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "y_test_predict = fitted_model.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"message\": \"9b60fd54-1820-45ae-a594-6136f217df34 - CacheDriver:Cached token is expired at 2021-01-30 22:13:50.025400.  Refreshing\"}\n",
      "{\"message\": \"9b60fd54-1820-45ae-a594-6136f217df34 - CacheDriver:Cached token is expired at 2021-01-30 22:13:50.025400.  Refreshing\"}\n",
      "{\"message\": \"9b60fd54-1820-45ae-a594-6136f217df34 - CacheDriver:Cached token is expired at 2021-01-30 22:13:50.025400.  Refreshing\"}\n",
      "{\"message\": \"9b60fd54-1820-45ae-a594-6136f217df34 - CacheDriver:Cached token is expired at 2021-01-30 22:13:50.025400.  Refreshing\"}\n",
      "{\"message\": \"9b60fd54-1820-45ae-a594-6136f217df34 - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"9b60fd54-1820-45ae-a594-6136f217df34 - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"9b60fd54-1820-45ae-a594-6136f217df34 - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"9b60fd54-1820-45ae-a594-6136f217df34 - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"9b60fd54-1820-45ae-a594-6136f217df34 - CacheDriver:Returning token refreshed after expiry.\"}\n",
      "{\"message\": \"9b60fd54-1820-45ae-a594-6136f217df34 - CacheDriver:Returning token refreshed after expiry.\"}\n",
      "{\"message\": \"9b60fd54-1820-45ae-a594-6136f217df34 - CacheDriver:Returning token refreshed after expiry.\"}\n",
      "{\"message\": \"9b60fd54-1820-45ae-a594-6136f217df34 - CacheDriver:Returning token refreshed after expiry.\"}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                          precision    recall  f1-score   support\n",
      "\n",
      "                  Benign     0.9904    0.9908    0.9906   2048549\n",
      "                     Bot     0.9992    0.9998    0.9995     43419\n",
      "        Brute Force -Web     0.9877    0.9524    0.9697        84\n",
      "        Brute Force -XSS     1.0000    0.9756    0.9877        41\n",
      "        DDOS attack-HOIC     0.9999    1.0000    0.9999     48951\n",
      "  DDoS attacks-LOIC-HTTP     1.0000    1.0000    1.0000     86440\n",
      "   DoS attacks-GoldenEye     0.9924    0.9994    0.9959      8184\n",
      "        DoS attacks-Hulk     0.9974    1.0000    0.9987      7658\n",
      "DoS attacks-SlowHTTPTest     0.7663    0.5213    0.6205     41976\n",
      "   DoS attacks-Slowloris     0.9789    1.0000    0.9893      2598\n",
      "          FTP-BruteForce     0.7186    0.8854    0.7934     58010\n",
      "           Infilteration     0.3417    0.3291    0.3353     29297\n",
      "           SQL Injection     1.0000    0.4615    0.6316        13\n",
      "          SSH-Bruteforce     0.9988    0.9998    0.9993     56208\n",
      "\n",
      "                accuracy                         0.9732   2431428\n",
      "               macro avg     0.9122    0.8654    0.8794   2431428\n",
      "            weighted avg     0.9731    0.9732    0.9725   2431428\n",
      "\n",
      "CPU times: user 3min 12s, sys: 2.33 s, total: 3min 14s\n",
      "Wall time: 3min 12s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "cr = classification_report(digits=4,\n",
    "                           y_true=y_test,\n",
    "                           y_pred=y_test_predict,\n",
    "                           output_dict=False)\n",
    "print(cr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "del test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test the model on the 2017 dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 8.76 s, sys: 4.88 s, total: 13.6 s\n",
      "Wall time: 1min 11s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "test_2017 = Dataset.get_by_name(\n",
    "            workspace=workspace,\n",
    "            name=dataset_2017_name,\n",
    "        ).to_pandas_dataframe()\n",
    "x_test_2017, y_test_2017 = test_2017.drop(labels=[\"Label\"], axis=1), test_2017[\"Label\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"message\": \"df6e07e1-03b9-4048-ad08-5548abb605ac - CacheDriver:Cached token is expired at 2021-01-30 23:09:23.734172.  Refreshing\"}\n",
      "{\"message\": \"df6e07e1-03b9-4048-ad08-5548abb605ac - CacheDriver:Cached token is expired at 2021-01-30 23:09:23.734172.  Refreshing\"}\n",
      "{\"message\": \"df6e07e1-03b9-4048-ad08-5548abb605ac - CacheDriver:Cached token is expired at 2021-01-30 23:09:23.734172.  Refreshing\"}\n",
      "{\"message\": \"df6e07e1-03b9-4048-ad08-5548abb605ac - CacheDriver:Cached token is expired at 2021-01-30 23:09:23.734172.  Refreshing\"}\n",
      "{\"message\": \"df6e07e1-03b9-4048-ad08-5548abb605ac - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"df6e07e1-03b9-4048-ad08-5548abb605ac - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"df6e07e1-03b9-4048-ad08-5548abb605ac - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"df6e07e1-03b9-4048-ad08-5548abb605ac - TokenRequest:Getting a new token from a refresh token\"}\n",
      "{\"message\": \"df6e07e1-03b9-4048-ad08-5548abb605ac - CacheDriver:Returning token refreshed after expiry.\"}\n",
      "{\"message\": \"df6e07e1-03b9-4048-ad08-5548abb605ac - CacheDriver:Returning token refreshed after expiry.\"}\n",
      "{\"message\": \"df6e07e1-03b9-4048-ad08-5548abb605ac - CacheDriver:Returning token refreshed after expiry.\"}\n",
      "{\"message\": \"df6e07e1-03b9-4048-ad08-5548abb605ac - CacheDriver:Returning token refreshed after expiry.\"}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 42min 25s, sys: 2min 26s, total: 44min 51s\n",
      "Wall time: 20min 31s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "y_test_2017_predict = fitted_model.predict(x_test_2017)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                          precision    recall  f1-score   support\n",
      "\n",
      "                  Benign     0.8239    0.9143    0.8668    953268\n",
      "                     Bot     0.0000    0.0000    0.0000      1956\n",
      "        Brute Force -Web     0.0000    0.0000    0.0000      1507\n",
      "        Brute Force -XSS     0.0000    0.0000    0.0000       635\n",
      "        DDOS attack-HOIC     0.0000    0.0000    0.0000         0\n",
      "   DoS attacks-GoldenEye     0.0336    0.0459    0.0388      7714\n",
      "        DoS attacks-Hulk     0.0029    0.0001    0.0002    164013\n",
      "DoS attacks-SlowHTTPTest     0.0000    0.0000    0.0000      2327\n",
      "   DoS attacks-Slowloris     0.2037    0.9610    0.3361      4154\n",
      "          FTP-BruteForce     0.0000    0.0000    0.0000      6443\n",
      "           Infilteration     0.0000    0.0000    0.0000        33\n",
      "           SQL Injection     0.0000    0.0000    0.0000        21\n",
      "          SSH-Bruteforce     0.0054    0.0201    0.0085      5885\n",
      "\n",
      "                accuracy                         0.7632   1147956\n",
      "               macro avg     0.0823    0.1493    0.0962   1147956\n",
      "            weighted avg     0.6856    0.7632    0.7213   1147956\n",
      "\n",
      "CPU times: user 1min 22s, sys: 845 ms, total: 1min 23s\n",
      "Wall time: 1min 22s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "cr = classification_report(digits=4,\n",
    "                           y_true=y_test_2017,\n",
    "                           y_pred=y_test_2017_predict,\n",
    "                           output_dict=False)\n",
    "print(cr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "del test_2017\n",
    "del x_test_2017"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": false,
     "source_hidden": false
    },
    "nteract": {
     "transient": {
      "deleting": false
     }
    }
   },
   "source": [
    "## Model Deployment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Register the best model into the workspace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 467 ms, sys: 4.68 ms, total: 472 ms\n",
      "Wall time: 4.29 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "model = package_utils.register_model(model_name=\"automl-jupyter\",\n",
    "                                     model_path=\"outputs/model.pkl\",\n",
    "                                     run=best_run)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'createdTime': '2021-01-30T22:11:46.123640+00:00', 'createdBy': {'userObjectId': 'b0e0dfc5-cdf2-4c0f-b904-9072f8246826', 'userPuId': '1003200111DF7E2A', 'userIss': 'https://sts.windows.net/660b3398-b80e-49d2-bc5b-ac1dc93b5254/', 'userTenantId': '660b3398-b80e-49d2-bc5b-ac1dc93b5254', 'userName': 'ODL_User 136633'}, 'description': None, 'id': 'automl-jupyter:1', 'mimeType': 'application/json', 'name': 'automl-jupyter', 'framework': 'Custom', 'frameworkVersion': None, 'tags': {}, 'properties': {}, 'unpack': False, 'url': 'aml://asset/4a80bfa1c9ba494e924baa21aebcdcef', 'version': 1, 'experimentName': 'jupyter-2021-01-29-09-02', 'runId': 'AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14', 'runDetails': 'Run(Experiment: jupyter-2021-01-29-09-02,\\nId: AutoML_ae1d998c-ec87-447d-bf70-328476f80063_14,\\nType: azureml.scriptrun,\\nStatus: Completed)', 'datasets': [], 'resourceConfiguration': None, 'sampleInputDatasetId': None, 'sampleOutputDatasetId': None}\n"
     ]
    }
   ],
   "source": [
    "print(model.serialize())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Deploy the registered model to an Azure Container instance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "{\"message\": \"main\", \"environment\": \"Environment(Name: automl-jupyter,\\nVersion: None)\"}\n",
      "{\"message\": \"main\", \"environment\": \"Environment(Name: automl-jupyter,\\nVersion: None)\"}\n",
      "{\"message\": \"main\", \"environment\": \"Environment(Name: automl-jupyter,\\nVersion: None)\"}\n",
      "{\"message\": \"main\", \"environment\": \"Environment(Name: automl-jupyter,\\nVersion: None)\"}\n",
      "{\"message\": \"main\", \"inference_config\": \"InferenceConfig(entry_script=model/deploy/score.py, runtime=None, conda_file=None, extra_docker_file_steps=None, source_directory=/app/nd00333, enable_gpu=None, base_image=None, base_image_registry=<azureml.core.container_registry.ContainerRegistry object at 0x7f68209bf290>)\"}\n",
      "{\"message\": \"main\", \"inference_config\": \"InferenceConfig(entry_script=model/deploy/score.py, runtime=None, conda_file=None, extra_docker_file_steps=None, source_directory=/app/nd00333, enable_gpu=None, base_image=None, base_image_registry=<azureml.core.container_registry.ContainerRegistry object at 0x7f68209bf290>)\"}\n",
      "{\"message\": \"main\", \"inference_config\": \"InferenceConfig(entry_script=model/deploy/score.py, runtime=None, conda_file=None, extra_docker_file_steps=None, source_directory=/app/nd00333, enable_gpu=None, base_image=None, base_image_registry=<azureml.core.container_registry.ContainerRegistry object at 0x7f68209bf290>)\"}\n",
      "{\"message\": \"main\", \"inference_config\": \"InferenceConfig(entry_script=model/deploy/score.py, runtime=None, conda_file=None, extra_docker_file_steps=None, source_directory=/app/nd00333, enable_gpu=None, base_image=None, base_image_registry=<azureml.core.container_registry.ContainerRegistry object at 0x7f68209bf290>)\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"Found the config file in: /app/config.json\"}\n",
      "{\"message\": \"main\", \"deployment_config\": \"<azureml.core.webservice.aci.AciServiceDeploymentConfiguration object at 0x7f68a4584410>\"}\n",
      "{\"message\": \"main\", \"deployment_config\": \"<azureml.core.webservice.aci.AciServiceDeploymentConfiguration object at 0x7f68a4584410>\"}\n",
      "{\"message\": \"main\", \"deployment_config\": \"<azureml.core.webservice.aci.AciServiceDeploymentConfiguration object at 0x7f68a4584410>\"}\n",
      "{\"message\": \"main\", \"deployment_config\": \"<azureml.core.webservice.aci.AciServiceDeploymentConfiguration object at 0x7f68a4584410>\"}\n",
      "{\"message\": \"main\", \"model\": \"Model(workspace=Workspace.create(name='quick-starts-ws-136633', subscription_id='a24a24d5-8d87-4c8a-99b6-91ed2d2df51f', resource_group='aml-quickstarts-136633'), name=automl-jupyter, id=automl-jupyter:1, version=1, tags={}, properties={})\"}\n",
      "{\"message\": \"main\", \"model\": \"Model(workspace=Workspace.create(name='quick-starts-ws-136633', subscription_id='a24a24d5-8d87-4c8a-99b6-91ed2d2df51f', resource_group='aml-quickstarts-136633'), name=automl-jupyter, id=automl-jupyter:1, version=1, tags={}, properties={})\"}\n",
      "{\"message\": \"main\", \"model\": \"Model(workspace=Workspace.create(name='quick-starts-ws-136633', subscription_id='a24a24d5-8d87-4c8a-99b6-91ed2d2df51f', resource_group='aml-quickstarts-136633'), name=automl-jupyter, id=automl-jupyter:1, version=1, tags={}, properties={})\"}\n",
      "{\"message\": \"main\", \"model\": \"Model(workspace=Workspace.create(name='quick-starts-ws-136633', subscription_id='a24a24d5-8d87-4c8a-99b6-91ed2d2df51f', resource_group='aml-quickstarts-136633'), name=automl-jupyter, id=automl-jupyter:1, version=1, tags={}, properties={})\"}\n",
      "{\"message\": \"main\", \"service\": \"AciWebservice(workspace=Workspace.create(name='quick-starts-ws-136633', subscription_id='a24a24d5-8d87-4c8a-99b6-91ed2d2df51f', resource_group='aml-quickstarts-136633'), name=automl-jupyter, image_id=None, compute_type=None, state=ACI, scoring_uri=Transitioning, tags=None, properties={}, created_by={})\"}\n",
      "{\"message\": \"main\", \"service\": \"AciWebservice(workspace=Workspace.create(name='quick-starts-ws-136633', subscription_id='a24a24d5-8d87-4c8a-99b6-91ed2d2df51f', resource_group='aml-quickstarts-136633'), name=automl-jupyter, image_id=None, compute_type=None, state=ACI, scoring_uri=Transitioning, tags=None, properties={}, created_by={})\"}\n",
      "{\"message\": \"main\", \"service\": \"AciWebservice(workspace=Workspace.create(name='quick-starts-ws-136633', subscription_id='a24a24d5-8d87-4c8a-99b6-91ed2d2df51f', resource_group='aml-quickstarts-136633'), name=automl-jupyter, image_id=None, compute_type=None, state=ACI, scoring_uri=Transitioning, tags=None, properties={}, created_by={})\"}\n",
      "{\"message\": \"main\", \"service\": \"AciWebservice(workspace=Workspace.create(name='quick-starts-ws-136633', subscription_id='a24a24d5-8d87-4c8a-99b6-91ed2d2df51f', resource_group='aml-quickstarts-136633'), name=automl-jupyter, image_id=None, compute_type=None, state=ACI, scoring_uri=Transitioning, tags=None, properties={}, created_by={})\"}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.12 s, sys: 169 ms, total: 2.29 s\n",
      "Wall time: 13 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "service = deploy_config.main(model_name=\"automl-jupyter\",\n",
    "                             deployment_name=\"automl-jupyter\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tips: You can try get_logs(): https://aka.ms/debugimage#dockerlog or local deployment: https://aka.ms/debugimage#debug-locally to debug if deployment takes longer than 10 minutes.\n",
      "Running....................................\n",
      "Succeeded\n",
      "ACI service creation operation finished, operation \"Succeeded\"\n",
      "CPU times: user 4.29 s, sys: 360 ms, total: 4.65 s\n",
      "Wall time: 3min 14s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "service.wait_for_deployment(show_output=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert service.state == \"Healthy\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test the service endpoint"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fetch the API keys of the service endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "primary_api_key, secondary_api_key = service.get_keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Retrive the scoring url of the service endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://fede3df5-b8eb-493f-ad37-45f325532bc0.southcentralus.azurecontainer.io/score\n"
     ]
    }
   ],
   "source": [
    "url = service.scoring_uri\n",
    "print(url)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Prepare a subset of the test dataset for submission to the service"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"data\": [{\"Flow Duration\": 3761843, \"TotLen Fwd Pkts\": 1441, \"TotLen Bwd Pkts\": 1731, \"Fwd Pkt Len Std\": 191, \"Bwd Pkt Len Max\": 1179, \"Bwd Pkt Len Std\": 405, \"Flow Byts/s\": 843, \"Flow Pkts/s\": 6, \"Flow IAT Max\": 953181, \"Bwd IAT Min\": 124510, \"Bwd Header Len\": 172, \"Pkt Len Max\": 1179, \"Pkt Len Std\": 279, \"RST Flag Cnt\": 1, \"PSH Flag Cnt\": 1, \"ECE Flag Cnt\": 1, \"Init Fwd Win Byts\": 8192, \"Init Bwd Win Byts\": 62644, \"Fwd Seg Size Min\": 20}]}"
     ]
    }
   ],
   "source": [
    "input_data = json.dumps({'data': x_test[0:1].to_dict(orient='records')})\n",
    "with open(\"data.json\", \"w\") as _f:\n",
    "    _f.write(input_data)\n",
    "!cat data.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Call the service using the input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"result\": [\"Benign\"]}\n"
     ]
    }
   ],
   "source": [
    "print(service.run(input_data))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Send a post request to the service endpoint using curl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"{\\\"result\\\": [\\\"Benign\\\"]}\"CPU times: user 31.4 ms, sys: 34.4 ms, total: 65.7 ms\n",
      "Wall time: 1.56 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "!curl -X POST \\\n",
    "      -H 'Content-Type: application/json' \\\n",
    "      -H \"Authorization: Bearer $secondary_api_key\" \\\n",
    "      --data @data.json $url"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Send a post request to the service endpoint programatically"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"result\": [\"Benign\"]}\n"
     ]
    }
   ],
   "source": [
    "# Set the content type\n",
    "headers = {\"Content-Type\": \"application/json\"}\n",
    "# If authentication is enabled, set the authorization header\n",
    "headers[\"Authorization\"] = f\"Bearer {secondary_api_key}\"\n",
    "\n",
    "resp = requests.post(url, input_data, headers=headers)\n",
    "print(resp.json())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "del x_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Print the service logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-01-30T22:14:51,530708557+00:00 - nginx/run \n",
      "2021-01-30T22:14:51,535531061+00:00 - gunicorn/run \n",
      "2021-01-30T22:14:51,536593362+00:00 - iot-server/run \n",
      "2021-01-30T22:14:51,539587765+00:00 - rsyslog/run \n",
      "EdgeHubConnectionString and IOTEDGE_IOTHUBHOSTNAME are not set. Exiting...\n",
      "2021-01-30T22:14:51,755418137+00:00 - iot-server/finish 1 0\n",
      "2021-01-30T22:14:51,756937239+00:00 - Exit code 1 is normal. Not restarting iot-server.\n",
      "Starting gunicorn 19.9.0\n",
      "Listening at: http://127.0.0.1:31311 (15)\n",
      "Using worker: sync\n",
      "worker timeout is set to 300\n",
      "Booting worker with pid: 42\n",
      "SPARK_HOME not set. Skipping PySpark Initialization.\n",
      "Initializing logger\n",
      "2021-01-30 22:14:52,598 | root | INFO | Starting up app insights client\n",
      "2021-01-30 22:14:52,598 | root | INFO | Starting up request id generator\n",
      "2021-01-30 22:14:52,598 | root | INFO | Starting up app insight hooks\n",
      "2021-01-30 22:14:52,598 | root | INFO | Invoking user's init function\n",
      "2021-01-30 22:14:55,232 | root | INFO | Users's init has completed successfully\n",
      "2021-01-30 22:14:55,234 | root | INFO | Skipping middleware: dbg_model_info as it's not enabled.\n",
      "2021-01-30 22:14:55,234 | root | INFO | Skipping middleware: dbg_resource_usage as it's not enabled.\n",
      "2021-01-30 22:14:55,235 | root | INFO | Scoring timeout is found from os.environ: 60000 ms\n",
      "2021-01-30 22:15:07,911 | root | INFO | 200\n",
      "127.0.0.1 - - [30/Jan/2021:22:15:07 +0000] \"GET /swagger.json HTTP/1.0\" 200 3430 \"-\" \"Go-http-client/1.1\"\n",
      "2021-01-30 22:15:14,338 | root | INFO | 200\n",
      "127.0.0.1 - - [30/Jan/2021:22:15:14 +0000] \"GET /swagger.json HTTP/1.0\" 200 3430 \"-\" \"Go-http-client/1.1\"\n",
      "2021-01-30 22:15:18,535 | root | INFO | Validation Request Content-Type\n",
      "2021-01-30 22:15:18,535 | root | INFO | Scoring Timer is set to 60.0 seconds\n",
      "2021-01-30 22:15:18,595 | root | INFO | 200\n",
      "127.0.0.1 - - [30/Jan/2021:22:15:18 +0000] \"POST /score HTTP/1.0\" 200 28 \"-\" \"python-requests/2.24.0\"\n",
      "2021-01-30 22:15:19,982 | root | INFO | Validation Request Content-Type\n",
      "2021-01-30 22:15:19,982 | root | INFO | Scoring Timer is set to 60.0 seconds\n",
      "2021-01-30 22:15:20,061 | root | INFO | 200\n",
      "127.0.0.1 - - [30/Jan/2021:22:15:20 +0000] \"POST /score HTTP/1.0\" 200 28 \"-\" \"curl/7.64.0\"\n",
      "2021-01-30 22:15:20,566 | root | INFO | Validation Request Content-Type\n",
      "2021-01-30 22:15:20,566 | root | INFO | Scoring Timer is set to 60.0 seconds\n",
      "2021-01-30 22:15:20,630 | root | INFO | 200\n",
      "127.0.0.1 - - [30/Jan/2021:22:15:20 +0000] \"POST /score HTTP/1.0\" 200 28 \"-\" \"python-requests/2.24.0\"\n",
      "2021-01-30 22:56:17,446 | root | INFO | 200\n",
      "127.0.0.1 - - [30/Jan/2021:22:56:17 +0000] \"GET /swagger.json HTTP/1.0\" 200 3430 \"-\" \"Go-http-client/1.1\"\n",
      "2021-01-30 22:56:19,535 | root | INFO | 200\n",
      "127.0.0.1 - - [30/Jan/2021:22:56:19 +0000] \"GET /swagger.json HTTP/1.0\" 200 3430 \"-\" \"Go-http-client/1.1\"\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(service.get_logs())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Delete the service endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "service.delete()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Delete the compute cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cluster_name starts-ws-136633\n"
     ]
    }
   ],
   "source": [
    "cluster_name = package_utils.trim_cluster_name(workspace.name)\n",
    "print(f\"cluster_name {cluster_name}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "compute_target starts-ws-136633 does not exist\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    compute_target = ComputeTarget(workspace=workspace, name=cluster_name)\n",
    "    compute_target.delete()\n",
    "    print(f\"compute_target {compute_target} is being deleted\")    \n",
    "except ComputeTargetException:\n",
    "    print(f\"compute_target {cluster_name} does not exist\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernel_info": {
   "name": "python3-azureml"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "nteract": {
   "version": "nteract-front-end@1.0.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}