datamole.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7b583695-7b59-4c8b-9f6c-754db3f658b4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: vaex in /opt/conda/lib/python3.7/site-packages (4.9.0)\n",
      "Requirement already satisfied: vaex-ml<0.18,>=0.17.0 in /opt/conda/lib/python3.7/site-packages (from vaex) (0.17.0)\n",
      "Requirement already satisfied: vaex-hdf5<0.13,>=0.12.1 in /opt/conda/lib/python3.7/site-packages (from vaex) (0.12.1)\n",
      "Requirement already satisfied: vaex-jupyter<0.8,>=0.7.0 in /opt/conda/lib/python3.7/site-packages (from vaex) (0.7.0)\n",
      "Requirement already satisfied: vaex-core<4.10,>=4.9.0 in /opt/conda/lib/python3.7/site-packages (from vaex) (4.9.0)\n",
      "Requirement already satisfied: vaex-server<0.9,>=0.8.1 in /opt/conda/lib/python3.7/site-packages (from vaex) (0.8.1)\n",
      "Requirement already satisfied: vaex-astro<0.10,>=0.9.1 in /opt/conda/lib/python3.7/site-packages (from vaex) (0.9.1)\n",
      "Requirement already satisfied: vaex-viz<0.6,>=0.5.1 in /opt/conda/lib/python3.7/site-packages (from vaex) (0.5.1)\n",
      "Requirement already satisfied: astropy in /opt/conda/lib/python3.7/site-packages (from vaex-astro<0.10,>=0.9.1->vaex) (4.3.1)\n",
      "Requirement already satisfied: progressbar2 in /opt/conda/lib/python3.7/site-packages (from vaex-core<4.10,>=4.9.0->vaex) (4.0.0)\n",
      "Requirement already satisfied: nest-asyncio>=1.3.3 in /opt/conda/lib/python3.7/site-packages (from vaex-core<4.10,>=4.9.0->vaex) (1.5.4)\n",
      "Requirement already satisfied: dask!=2022.4.0 in /opt/conda/lib/python3.7/site-packages (from vaex-core<4.10,>=4.9.0->vaex) (2022.2.0)\n",
      "Requirement already satisfied: blake3 in /opt/conda/lib/python3.7/site-packages (from vaex-core<4.10,>=4.9.0->vaex) (0.3.1)\n",
      "Requirement already satisfied: rich in /opt/conda/lib/python3.7/site-packages (from vaex-core<4.10,>=4.9.0->vaex) (12.2.0)\n",
      "Requirement already satisfied: future>=0.15.2 in /opt/conda/lib/python3.7/site-packages (from vaex-core<4.10,>=4.9.0->vaex) (0.18.2)\n",
      "Requirement already satisfied: pandas in /opt/conda/lib/python3.7/site-packages (from vaex-core<4.10,>=4.9.0->vaex) (1.3.5)\n",
      "Requirement already satisfied: pyyaml in /opt/conda/lib/python3.7/site-packages (from vaex-core<4.10,>=4.9.0->vaex) (6.0)\n",
      "Requirement already satisfied: cloudpickle in /opt/conda/lib/python3.7/site-packages (from vaex-core<4.10,>=4.9.0->vaex) (2.0.0)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.7/site-packages (from vaex-core<4.10,>=4.9.0->vaex) (3.6.0)\n",
      "Requirement already satisfied: pydantic>=1.8.0 in /opt/conda/lib/python3.7/site-packages (from vaex-core<4.10,>=4.9.0->vaex) (1.9.0)\n",
      "Requirement already satisfied: aplus in /opt/conda/lib/python3.7/site-packages (from vaex-core<4.10,>=4.9.0->vaex) (0.11.0)\n",
      "Requirement already satisfied: numpy>=1.16 in /opt/conda/lib/python3.7/site-packages (from vaex-core<4.10,>=4.9.0->vaex) (1.19.5)\n",
      "Requirement already satisfied: frozendict!=2.2.0 in /opt/conda/lib/python3.7/site-packages (from vaex-core<4.10,>=4.9.0->vaex) (2.3.1)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.7/site-packages (from vaex-core<4.10,>=4.9.0->vaex) (1.16.0)\n",
      "Requirement already satisfied: pyarrow>=3.0 in /opt/conda/lib/python3.7/site-packages (from vaex-core<4.10,>=4.9.0->vaex) (7.0.0)\n",
      "Requirement already satisfied: tabulate>=0.8.3 in /opt/conda/lib/python3.7/site-packages (from vaex-core<4.10,>=4.9.0->vaex) (0.8.9)\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.7/site-packages (from vaex-core<4.10,>=4.9.0->vaex) (2.27.1)\n",
      "Requirement already satisfied: h5py>=2.9 in /opt/conda/lib/python3.7/site-packages (from vaex-hdf5<0.13,>=0.12.1->vaex) (3.6.0)\n",
      "Requirement already satisfied: ipyvuetify<2,>=1.2.2 in /opt/conda/lib/python3.7/site-packages (from vaex-jupyter<0.8,>=0.7.0->vaex) (1.8.2)\n",
      "Requirement already satisfied: ipyvolume>=0.4 in /opt/conda/lib/python3.7/site-packages (from vaex-jupyter<0.8,>=0.7.0->vaex) (0.5.2)\n",
      "Requirement already satisfied: bqplot>=0.10.1 in /opt/conda/lib/python3.7/site-packages (from vaex-jupyter<0.8,>=0.7.0->vaex) (0.12.33)\n",
      "Requirement already satisfied: xarray in /opt/conda/lib/python3.7/site-packages (from vaex-jupyter<0.8,>=0.7.0->vaex) (0.20.2)\n",
      "Requirement already satisfied: ipyleaflet in /opt/conda/lib/python3.7/site-packages (from vaex-jupyter<0.8,>=0.7.0->vaex) (0.15.0)\n",
      "Requirement already satisfied: ipympl in /opt/conda/lib/python3.7/site-packages (from vaex-jupyter<0.8,>=0.7.0->vaex) (0.8.8)\n",
      "Requirement already satisfied: numba in /opt/conda/lib/python3.7/site-packages (from vaex-ml<0.18,>=0.17.0->vaex) (0.55.1)\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.7/site-packages (from vaex-ml<0.18,>=0.17.0->vaex) (3.0.3)\n",
      "Requirement already satisfied: traitlets in /opt/conda/lib/python3.7/site-packages (from vaex-ml<0.18,>=0.17.0->vaex) (5.1.1)\n",
      "Requirement already satisfied: tornado>4.1 in /opt/conda/lib/python3.7/site-packages (from vaex-server<0.9,>=0.8.1->vaex) (6.1)\n",
      "Requirement already satisfied: cachetools in /opt/conda/lib/python3.7/site-packages (from vaex-server<0.9,>=0.8.1->vaex) (5.0.0)\n",
      "Requirement already satisfied: fastapi in /opt/conda/lib/python3.7/site-packages (from vaex-server<0.9,>=0.8.1->vaex) (0.75.1)\n",
      "Requirement already satisfied: uvicorn[standard] in /opt/conda/lib/python3.7/site-packages (from vaex-server<0.9,>=0.8.1->vaex) (0.17.6)\n",
      "Requirement already satisfied: pillow in /opt/conda/lib/python3.7/site-packages (from vaex-viz<0.6,>=0.5.1->vaex) (9.0.1)\n",
      "Requirement already satisfied: matplotlib>=1.3.1 in /opt/conda/lib/python3.7/site-packages (from vaex-viz<0.6,>=0.5.1->vaex) (3.5.1)\n",
      "Requirement already satisfied: traittypes>=0.0.6 in /opt/conda/lib/python3.7/site-packages (from bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (0.2.1)\n",
      "Requirement already satisfied: ipywidgets>=7.5.0 in /opt/conda/lib/python3.7/site-packages (from bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (7.6.5)\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.7/site-packages (from dask!=2022.4.0->vaex-core<4.10,>=4.9.0->vaex) (21.3)\n",
      "Requirement already satisfied: partd>=0.3.10 in /opt/conda/lib/python3.7/site-packages (from dask!=2022.4.0->vaex-core<4.10,>=4.9.0->vaex) (1.2.0)\n",
      "Requirement already satisfied: fsspec>=0.6.0 in /opt/conda/lib/python3.7/site-packages (from dask!=2022.4.0->vaex-core<4.10,>=4.9.0->vaex) (2022.2.0)\n",
      "Requirement already satisfied: toolz>=0.8.2 in /opt/conda/lib/python3.7/site-packages (from dask!=2022.4.0->vaex-core<4.10,>=4.9.0->vaex) (0.11.2)\n",
      "Requirement already satisfied: cached-property in /opt/conda/lib/python3.7/site-packages (from h5py>=2.9->vaex-hdf5<0.13,>=0.12.1->vaex) (1.5.2)\n",
      "Requirement already satisfied: pythreejs>=1.0.0 in /opt/conda/lib/python3.7/site-packages (from ipyvolume>=0.4->vaex-jupyter<0.8,>=0.7.0->vaex) (2.3.0)\n",
      "Requirement already satisfied: ipywebrtc in /opt/conda/lib/python3.7/site-packages (from ipyvolume>=0.4->vaex-jupyter<0.8,>=0.7.0->vaex) (0.6.0)\n",
      "Requirement already satisfied: ipyvue<2,>=1.5 in /opt/conda/lib/python3.7/site-packages (from ipyvuetify<2,>=1.2.2->vaex-jupyter<0.8,>=0.7.0->vaex) (1.7.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.7/site-packages (from matplotlib>=1.3.1->vaex-viz<0.6,>=0.5.1->vaex) (1.4.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /opt/conda/lib/python3.7/site-packages (from matplotlib>=1.3.1->vaex-viz<0.6,>=0.5.1->vaex) (4.30.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /opt/conda/lib/python3.7/site-packages (from matplotlib>=1.3.1->vaex-viz<0.6,>=0.5.1->vaex) (2.8.2)\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in /opt/conda/lib/python3.7/site-packages (from matplotlib>=1.3.1->vaex-viz<0.6,>=0.5.1->vaex) (3.0.7)\n",
      "Requirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.7/site-packages (from matplotlib>=1.3.1->vaex-viz<0.6,>=0.5.1->vaex) (0.11.0)\n",
      "Requirement already satisfied: pytz>=2017.3 in /opt/conda/lib/python3.7/site-packages (from pandas->vaex-core<4.10,>=4.9.0->vaex) (2021.3)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.7/site-packages (from pydantic>=1.8.0->vaex-core<4.10,>=4.9.0->vaex) (4.1.1)\n",
      "Requirement already satisfied: pyerfa>=1.7.3 in /opt/conda/lib/python3.7/site-packages (from astropy->vaex-astro<0.10,>=0.9.1->vaex) (2.0.0.1)\n",
      "Requirement already satisfied: importlib-metadata in /opt/conda/lib/python3.7/site-packages (from astropy->vaex-astro<0.10,>=0.9.1->vaex) (4.11.3)\n",
      "Requirement already satisfied: starlette==0.17.1 in /opt/conda/lib/python3.7/site-packages (from fastapi->vaex-server<0.9,>=0.8.1->vaex) (0.17.1)\n",
      "Requirement already satisfied: anyio<4,>=3.0.0 in /opt/conda/lib/python3.7/site-packages (from starlette==0.17.1->fastapi->vaex-server<0.9,>=0.8.1->vaex) (3.5.0)\n",
      "Requirement already satisfied: xyzservices>=2021.8.1 in /opt/conda/lib/python3.7/site-packages (from ipyleaflet->vaex-jupyter<0.8,>=0.7.0->vaex) (2022.3.0)\n",
      "Requirement already satisfied: ipython-genutils in /opt/conda/lib/python3.7/site-packages (from ipympl->vaex-jupyter<0.8,>=0.7.0->vaex) (0.2.0)\n",
      "Requirement already satisfied: ipython<9 in /opt/conda/lib/python3.7/site-packages (from ipympl->vaex-jupyter<0.8,>=0.7.0->vaex) (7.32.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.7/site-packages (from jinja2->vaex-ml<0.18,>=0.17.0->vaex) (2.1.1)\n",
      "Requirement already satisfied: llvmlite<0.39,>=0.38.0rc1 in /opt/conda/lib/python3.7/site-packages (from numba->vaex-ml<0.18,>=0.17.0->vaex) (0.38.0)\n",
      "Requirement already satisfied: setuptools in /opt/conda/lib/python3.7/site-packages (from numba->vaex-ml<0.18,>=0.17.0->vaex) (59.8.0)\n",
      "Requirement already satisfied: python-utils>=3.0.0 in /opt/conda/lib/python3.7/site-packages (from progressbar2->vaex-core<4.10,>=4.9.0->vaex) (3.1.0)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests->vaex-core<4.10,>=4.9.0->vaex) (1.26.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests->vaex-core<4.10,>=4.9.0->vaex) (2021.10.8)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /opt/conda/lib/python3.7/site-packages (from requests->vaex-core<4.10,>=4.9.0->vaex) (2.0.12)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests->vaex-core<4.10,>=4.9.0->vaex) (3.3)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.6.0 in /opt/conda/lib/python3.7/site-packages (from rich->vaex-core<4.10,>=4.9.0->vaex) (2.11.2)\n",
      "Requirement already satisfied: commonmark<0.10.0,>=0.9.0 in /opt/conda/lib/python3.7/site-packages (from rich->vaex-core<4.10,>=4.9.0->vaex) (0.9.1)\n",
      "Requirement already satisfied: h11>=0.8 in /opt/conda/lib/python3.7/site-packages (from uvicorn[standard]->vaex-server<0.9,>=0.8.1->vaex) (0.13.0)\n",
      "Requirement already satisfied: click>=7.0 in /opt/conda/lib/python3.7/site-packages (from uvicorn[standard]->vaex-server<0.9,>=0.8.1->vaex) (8.0.4)\n",
      "Requirement already satisfied: asgiref>=3.4.0 in /opt/conda/lib/python3.7/site-packages (from uvicorn[standard]->vaex-server<0.9,>=0.8.1->vaex) (3.5.0)\n",
      "Requirement already satisfied: httptools>=0.4.0 in /opt/conda/lib/python3.7/site-packages (from uvicorn[standard]->vaex-server<0.9,>=0.8.1->vaex) (0.4.0)\n",
      "Requirement already satisfied: websockets>=10.0 in /opt/conda/lib/python3.7/site-packages (from uvicorn[standard]->vaex-server<0.9,>=0.8.1->vaex) (10.2)\n",
      "Requirement already satisfied: python-dotenv>=0.13 in /opt/conda/lib/python3.7/site-packages (from uvicorn[standard]->vaex-server<0.9,>=0.8.1->vaex) (0.20.0)\n",
      "Requirement already satisfied: uvloop!=0.15.0,!=0.15.1,>=0.14.0 in /opt/conda/lib/python3.7/site-packages (from uvicorn[standard]->vaex-server<0.9,>=0.8.1->vaex) (0.16.0)\n",
      "Requirement already satisfied: watchgod>=0.6 in /opt/conda/lib/python3.7/site-packages (from uvicorn[standard]->vaex-server<0.9,>=0.8.1->vaex) (0.8.2)\n",
      "Requirement already satisfied: pexpect>4.3 in /opt/conda/lib/python3.7/site-packages (from ipython<9->ipympl->vaex-jupyter<0.8,>=0.7.0->vaex) (4.8.0)\n",
      "Requirement already satisfied: decorator in /opt/conda/lib/python3.7/site-packages (from ipython<9->ipympl->vaex-jupyter<0.8,>=0.7.0->vaex) (5.1.1)\n",
      "Requirement already satisfied: backcall in /opt/conda/lib/python3.7/site-packages (from ipython<9->ipympl->vaex-jupyter<0.8,>=0.7.0->vaex) (0.2.0)\n",
      "Requirement already satisfied: jedi>=0.16 in /opt/conda/lib/python3.7/site-packages (from ipython<9->ipympl->vaex-jupyter<0.8,>=0.7.0->vaex) (0.18.1)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from ipython<9->ipympl->vaex-jupyter<0.8,>=0.7.0->vaex) (3.0.27)\n",
      "Requirement already satisfied: matplotlib-inline in /opt/conda/lib/python3.7/site-packages (from ipython<9->ipympl->vaex-jupyter<0.8,>=0.7.0->vaex) (0.1.3)\n",
      "Requirement already satisfied: pickleshare in /opt/conda/lib/python3.7/site-packages (from ipython<9->ipympl->vaex-jupyter<0.8,>=0.7.0->vaex) (0.7.5)\n",
      "Requirement already satisfied: ipykernel>=4.5.1 in /opt/conda/lib/python3.7/site-packages (from ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (6.9.2)\n",
      "Requirement already satisfied: jupyterlab-widgets>=1.0.0 in /opt/conda/lib/python3.7/site-packages (from ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (1.0.2)\n",
      "Requirement already satisfied: widgetsnbextension~=3.5.0 in /opt/conda/lib/python3.7/site-packages (from ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (3.5.2)\n",
      "Requirement already satisfied: nbformat>=4.2.0 in /opt/conda/lib/python3.7/site-packages (from ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (5.2.0)\n",
      "Requirement already satisfied: locket in /opt/conda/lib/python3.7/site-packages (from partd>=0.3.10->dask!=2022.4.0->vaex-core<4.10,>=4.9.0->vaex) (0.2.1)\n",
      "Requirement already satisfied: ipydatawidgets>=1.1.1 in /opt/conda/lib/python3.7/site-packages (from pythreejs>=1.0.0->ipyvolume>=0.4->vaex-jupyter<0.8,>=0.7.0->vaex) (4.2.0)\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->astropy->vaex-astro<0.10,>=0.9.1->vaex) (3.7.0)\n",
      "Requirement already satisfied: sniffio>=1.1 in /opt/conda/lib/python3.7/site-packages (from anyio<4,>=3.0.0->starlette==0.17.1->fastapi->vaex-server<0.9,>=0.8.1->vaex) (1.2.0)\n",
      "Requirement already satisfied: jupyter-client<8.0 in /opt/conda/lib/python3.7/site-packages (from ipykernel>=4.5.1->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (7.1.2)\n",
      "Requirement already satisfied: debugpy<2.0,>=1.0.0 in /opt/conda/lib/python3.7/site-packages (from ipykernel>=4.5.1->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (1.5.1)\n",
      "Requirement already satisfied: psutil in /opt/conda/lib/python3.7/site-packages (from ipykernel>=4.5.1->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (5.9.0)\n",
      "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /opt/conda/lib/python3.7/site-packages (from jedi>=0.16->ipython<9->ipympl->vaex-jupyter<0.8,>=0.7.0->vaex) (0.8.3)\n",
      "Requirement already satisfied: jsonschema!=2.5.0,>=2.4 in /opt/conda/lib/python3.7/site-packages (from nbformat>=4.2.0->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (4.4.0)\n",
      "Requirement already satisfied: jupyter-core in /opt/conda/lib/python3.7/site-packages (from nbformat>=4.2.0->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (4.9.2)\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /opt/conda/lib/python3.7/site-packages (from pexpect>4.3->ipython<9->ipympl->vaex-jupyter<0.8,>=0.7.0->vaex) (0.7.0)\n",
      "Requirement already satisfied: wcwidth in /opt/conda/lib/python3.7/site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython<9->ipympl->vaex-jupyter<0.8,>=0.7.0->vaex) (0.2.5)\n",
      "Requirement already satisfied: notebook>=4.4.1 in /opt/conda/lib/python3.7/site-packages (from widgetsnbextension~=3.5.0->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (6.4.10)\n",
      "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /opt/conda/lib/python3.7/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.2.0->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (0.18.1)\n",
      "Requirement already satisfied: attrs>=17.4.0 in /opt/conda/lib/python3.7/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.2.0->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (21.4.0)\n",
      "Requirement already satisfied: importlib-resources>=1.4.0 in /opt/conda/lib/python3.7/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.2.0->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (5.4.0)\n",
      "Requirement already satisfied: entrypoints in /opt/conda/lib/python3.7/site-packages (from jupyter-client<8.0->ipykernel>=4.5.1->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (0.4)\n",
      "Requirement already satisfied: pyzmq>=13 in /opt/conda/lib/python3.7/site-packages (from jupyter-client<8.0->ipykernel>=4.5.1->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (22.3.0)\n",
      "Requirement already satisfied: nbconvert>=5 in /opt/conda/lib/python3.7/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (6.4.4)\n",
      "Requirement already satisfied: terminado>=0.8.3 in /opt/conda/lib/python3.7/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (0.13.3)\n",
      "Requirement already satisfied: Send2Trash>=1.8.0 in /opt/conda/lib/python3.7/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (1.8.0)\n",
      "Requirement already satisfied: argon2-cffi in /opt/conda/lib/python3.7/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (21.3.0)\n",
      "Requirement already satisfied: prometheus-client in /opt/conda/lib/python3.7/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (0.13.1)\n",
      "Requirement already satisfied: beautifulsoup4 in /opt/conda/lib/python3.7/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (4.10.0)\n",
      "Requirement already satisfied: defusedxml in /opt/conda/lib/python3.7/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (0.7.1)\n",
      "Requirement already satisfied: pandocfilters>=1.4.1 in /opt/conda/lib/python3.7/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (1.5.0)\n",
      "Requirement already satisfied: jupyterlab-pygments in /opt/conda/lib/python3.7/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (0.1.2)\n",
      "Requirement already satisfied: nbclient<0.6.0,>=0.5.0 in /opt/conda/lib/python3.7/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (0.5.13)\n",
      "Requirement already satisfied: mistune<2,>=0.8.1 in /opt/conda/lib/python3.7/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (0.8.4)\n",
      "Requirement already satisfied: testpath in /opt/conda/lib/python3.7/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (0.6.0)\n",
      "Requirement already satisfied: bleach in /opt/conda/lib/python3.7/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (4.1.0)\n",
      "Requirement already satisfied: argon2-cffi-bindings in /opt/conda/lib/python3.7/site-packages (from argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (21.2.0)\n",
      "Requirement already satisfied: cffi>=1.0.1 in /opt/conda/lib/python3.7/site-packages (from argon2-cffi-bindings->argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (1.15.0)\n",
      "Requirement already satisfied: soupsieve>1.2 in /opt/conda/lib/python3.7/site-packages (from beautifulsoup4->nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (2.3.1)\n",
      "Requirement already satisfied: webencodings in /opt/conda/lib/python3.7/site-packages (from bleach->nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (0.5.1)\n",
      "Requirement already satisfied: pycparser in /opt/conda/lib/python3.7/site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.0->bqplot>=0.10.1->vaex-jupyter<0.8,>=0.7.0->vaex) (2.21)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install vaex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8a531a5e-f65f-41d1-9340-f2aa3011fa08",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MachineId</th>\n",
       "      <th>MeasurementId</th>\n",
       "      <th>PumpFailed</th>\n",
       "      <th>SlowStart</th>\n",
       "      <th>SlowEnd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_0_0</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0_0_1</td>\n",
       "      <td>-1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0_0_2</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0_0_3</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0_1_0</td>\n",
       "      <td>-1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27380</th>\n",
       "      <td>10_0_2</td>\n",
       "      <td>8833</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27381</th>\n",
       "      <td>10_0_3</td>\n",
       "      <td>8833</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27382</th>\n",
       "      <td>10_0_0</td>\n",
       "      <td>8834</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27383</th>\n",
       "      <td>10_0_1</td>\n",
       "      <td>8834</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27384</th>\n",
       "      <td>10_0_3</td>\n",
       "      <td>8834</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>27385 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      MachineId  MeasurementId PumpFailed SlowStart SlowEnd\n",
       "0         0_0_0              0      False     False   False\n",
       "1         0_0_1             -1        NaN       NaN     NaN\n",
       "2         0_0_2              0      False     False   False\n",
       "3         0_0_3              0       True     False   False\n",
       "4         0_1_0             -1        NaN       NaN     NaN\n",
       "...         ...            ...        ...       ...     ...\n",
       "27380    10_0_2           8833      False     False   False\n",
       "27381    10_0_3           8833      False     False   False\n",
       "27382    10_0_0           8834      False     False   False\n",
       "27383    10_0_1           8834      False     False    True\n",
       "27384    10_0_3           8834      False     False   False\n",
       "\n",
       "[27385 rows x 5 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd \n",
    "import vaex\n",
    "import numpy as np\n",
    "import vaex.ml\n",
    "from vaex.ml.sklearn import Predictor\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "import pylab as plt\n",
    "df1 = pd.read_csv('labels.csv')\n",
    "df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6f213f7d-277f-482b-820e-261ce74263f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MachineId        0\n",
       "MeasurementId    0\n",
       "PumpFailed       0\n",
       "SlowStart        0\n",
       "SlowEnd          0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1 = df1.fillna(method='bfill',axis=0).fillna(0)\n",
    "df1.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "54d2a4f3-3ba5-463e-a117-34f970571b93",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MachineId        int64\n",
       "MeasurementId    int64\n",
       "PumpFailed       int64\n",
       "SlowStart        int64\n",
       "SlowEnd          int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1 = df1.astype(int)\n",
    "df1.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5fb94b12-44d1-41f4-8114-0778d5afa0c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MachineId</th>\n",
       "      <th>MeasurementId</th>\n",
       "      <th>PumpFailed</th>\n",
       "      <th>SlowStart</th>\n",
       "      <th>SlowEnd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27380</th>\n",
       "      <td>1002</td>\n",
       "      <td>8833</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27381</th>\n",
       "      <td>1003</td>\n",
       "      <td>8833</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27382</th>\n",
       "      <td>1000</td>\n",
       "      <td>8834</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27383</th>\n",
       "      <td>1001</td>\n",
       "      <td>8834</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27384</th>\n",
       "      <td>1003</td>\n",
       "      <td>8834</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>27385 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       MachineId  MeasurementId  PumpFailed  SlowStart  SlowEnd\n",
       "0              0              0           0          0        0\n",
       "1              1             -1           0          0        0\n",
       "2              2              0           0          0        0\n",
       "3              3              0           1          0        0\n",
       "4             10             -1           1          0        0\n",
       "...          ...            ...         ...        ...      ...\n",
       "27380       1002           8833           0          0        0\n",
       "27381       1003           8833           0          0        0\n",
       "27382       1000           8834           0          0        0\n",
       "27383       1001           8834           0          0        1\n",
       "27384       1003           8834           0          0        0\n",
       "\n",
       "[27385 rows x 5 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b621bcdb-ae5d-40d3-82a8-7fcb277d53e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_full = vaex.from_pandas(df1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d1152158-cb9f-4858-822b-21e49e6a6dcd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>#                                 </th><th>MachineId  </th><th>MeasurementId  </th><th>PumpFailed  </th><th>SlowStart  </th><th>SlowEnd  </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td><i style='opacity: 0.6'>0</i>     </td><td>0          </td><td>0              </td><td>0           </td><td>0          </td><td>0        </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>1</i>     </td><td>1          </td><td>-1             </td><td>0           </td><td>0          </td><td>0        </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>2</i>     </td><td>2          </td><td>0              </td><td>0           </td><td>0          </td><td>0        </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3</i>     </td><td>3          </td><td>0              </td><td>1           </td><td>0          </td><td>0        </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>4</i>     </td><td>10         </td><td>-1             </td><td>1           </td><td>0          </td><td>0        </td></tr>\n",
       "<tr><td>...                               </td><td>...        </td><td>...            </td><td>...         </td><td>...        </td><td>...      </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>27,380</i></td><td>1002       </td><td>8833           </td><td>0           </td><td>0          </td><td>0        </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>27,381</i></td><td>1003       </td><td>8833           </td><td>0           </td><td>0          </td><td>0        </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>27,382</i></td><td>1000       </td><td>8834           </td><td>0           </td><td>0          </td><td>0        </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>27,383</i></td><td>1001       </td><td>8834           </td><td>0           </td><td>0          </td><td>1        </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>27,384</i></td><td>1003       </td><td>8834           </td><td>0           </td><td>0          </td><td>0        </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "#       MachineId    MeasurementId    PumpFailed    SlowStart    SlowEnd\n",
       "0       0            0                0             0            0\n",
       "1       1            -1               0             0            0\n",
       "2       2            0                0             0            0\n",
       "3       3            0                1             0            0\n",
       "4       10           -1               1             0            0\n",
       "...     ...          ...              ...           ...          ...\n",
       "27,380  1002         8833             0             0            0\n",
       "27,381  1003         8833             0             0            0\n",
       "27,382  1000         8834             0             0            0\n",
       "27,383  1001         8834             0             0            1\n",
       "27,384  1003         8834             0             0            0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_full"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b4e7dd1a-78a6-46ff-bdaa-74e50da604b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = pd.read_parquet('data.parquet')\n",
    "df2.to_csv('data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e8f69d94-829a-4e73-b913-d02547689760",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MachineId</th>\n",
       "      <th>MeasurementId</th>\n",
       "      <th>Pressure</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_0_0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0_0_0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0_0_0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0_0_0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0_0_0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18466359</th>\n",
       "      <td>10_0_3</td>\n",
       "      <td>8834</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18466360</th>\n",
       "      <td>10_0_3</td>\n",
       "      <td>8834</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18466361</th>\n",
       "      <td>10_0_3</td>\n",
       "      <td>8834</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18466362</th>\n",
       "      <td>10_0_3</td>\n",
       "      <td>8834</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18466363</th>\n",
       "      <td>10_0_3</td>\n",
       "      <td>8834</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>18466364 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         MachineId  MeasurementId  Pressure\n",
       "0            0_0_0              0       0.0\n",
       "1            0_0_0              0       0.0\n",
       "2            0_0_0              0       0.0\n",
       "3            0_0_0              0       0.0\n",
       "4            0_0_0              0       0.0\n",
       "...            ...            ...       ...\n",
       "18466359    10_0_3           8834       0.0\n",
       "18466360    10_0_3           8834       0.0\n",
       "18466361    10_0_3           8834       0.0\n",
       "18466362    10_0_3           8834       0.0\n",
       "18466363    10_0_3           8834       0.0\n",
       "\n",
       "[18466364 rows x 3 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f348a3b5-ef48-471e-8b35-2b2d441c618c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MachineId         object\n",
       "MeasurementId      int64\n",
       "Pressure         float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9b54c5dc-9379-441e-8c62-2463867242a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MachineId        0\n",
       "MeasurementId    0\n",
       "Pressure         0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "99d67dbc-6a24-4437-9f92-3041e93ff224",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MachineId        int64\n",
       "MeasurementId    int64\n",
       "Pressure         int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2 = df2.astype(int)\n",
    "df2.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "73969d04-957e-4475-8ba9-389401eec0a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MachineId</th>\n",
       "      <th>MeasurementId</th>\n",
       "      <th>Pressure</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18466359</th>\n",
       "      <td>1003</td>\n",
       "      <td>8834</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18466360</th>\n",
       "      <td>1003</td>\n",
       "      <td>8834</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18466361</th>\n",
       "      <td>1003</td>\n",
       "      <td>8834</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18466362</th>\n",
       "      <td>1003</td>\n",
       "      <td>8834</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18466363</th>\n",
       "      <td>1003</td>\n",
       "      <td>8834</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>18466364 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          MachineId  MeasurementId  Pressure\n",
       "0                 0              0         0\n",
       "1                 0              0         0\n",
       "2                 0              0         0\n",
       "3                 0              0         0\n",
       "4                 0              0         0\n",
       "...             ...            ...       ...\n",
       "18466359       1003           8834         0\n",
       "18466360       1003           8834         0\n",
       "18466361       1003           8834         0\n",
       "18466362       1003           8834         0\n",
       "18466363       1003           8834         0\n",
       "\n",
       "[18466364 rows x 3 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b0d8632e-ca10-4693-a1cc-9e2caec9b96f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_full_2 = vaex.from_pandas(df2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e6b4d127-9f82-428b-85ab-74436e272a60",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>#                                     </th><th>MachineId  </th><th>MeasurementId  </th><th>PumpFailed  </th><th>SlowStart  </th><th>SlowEnd  </th><th>Pressure  </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td><i style='opacity: 0.6'>0</i>         </td><td>0          </td><td>0              </td><td>0           </td><td>0          </td><td>0        </td><td>--        </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>1</i>         </td><td>1          </td><td>-1             </td><td>0           </td><td>0          </td><td>0        </td><td>--        </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>2</i>         </td><td>2          </td><td>0              </td><td>0           </td><td>0          </td><td>0        </td><td>--        </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3</i>         </td><td>3          </td><td>0              </td><td>1           </td><td>0          </td><td>0        </td><td>--        </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>4</i>         </td><td>10         </td><td>-1             </td><td>1           </td><td>0          </td><td>0        </td><td>--        </td></tr>\n",
       "<tr><td>...                                   </td><td>...        </td><td>...            </td><td>...         </td><td>...        </td><td>...      </td><td>...       </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>18,493,744</i></td><td>1003       </td><td>8834           </td><td>--          </td><td>--         </td><td>--       </td><td>0         </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>18,493,745</i></td><td>1003       </td><td>8834           </td><td>--          </td><td>--         </td><td>--       </td><td>0         </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>18,493,746</i></td><td>1003       </td><td>8834           </td><td>--          </td><td>--         </td><td>--       </td><td>0         </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>18,493,747</i></td><td>1003       </td><td>8834           </td><td>--          </td><td>--         </td><td>--       </td><td>0         </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>18,493,748</i></td><td>1003       </td><td>8834           </td><td>--          </td><td>--         </td><td>--       </td><td>0         </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "#           MachineId    MeasurementId    PumpFailed    SlowStart    SlowEnd    Pressure\n",
       "0           0            0                0             0            0          --\n",
       "1           1            -1               0             0            0          --\n",
       "2           2            0                0             0            0          --\n",
       "3           3            0                1             0            0          --\n",
       "4           10           -1               1             0            0          --\n",
       "...         ...          ...              ...           ...          ...        ...\n",
       "18,493,744  1003         8834             --            --           --         0\n",
       "18,493,745  1003         8834             --            --           --         0\n",
       "18,493,746  1003         8834             --            --           --         0\n",
       "18,493,747  1003         8834             --            --           --         0\n",
       "18,493,748  1003         8834             --            --           --         0"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#The place where I made a \"mistake\"\n",
    "data = vaex.concat([df_full,df_full_2],resolver='flexible') \n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "831b8e2b-0a0d-4d76-9b1c-3c70c6e16b66",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MachineId</th>\n",
       "      <th>MeasurementId</th>\n",
       "      <th>PumpFailed</th>\n",
       "      <th>SlowStart</th>\n",
       "      <th>SlowEnd</th>\n",
       "      <th>Pressure</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10</td>\n",
       "      <td>-1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18493744</th>\n",
       "      <td>1003</td>\n",
       "      <td>8834</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18493745</th>\n",
       "      <td>1003</td>\n",
       "      <td>8834</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18493746</th>\n",
       "      <td>1003</td>\n",
       "      <td>8834</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18493747</th>\n",
       "      <td>1003</td>\n",
       "      <td>8834</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18493748</th>\n",
       "      <td>1003</td>\n",
       "      <td>8834</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>18493749 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          MachineId  MeasurementId  PumpFailed  SlowStart  SlowEnd  Pressure\n",
       "0                 0              0         0.0        0.0      0.0       NaN\n",
       "1                 1             -1         0.0        0.0      0.0       NaN\n",
       "2                 2              0         0.0        0.0      0.0       NaN\n",
       "3                 3              0         1.0        0.0      0.0       NaN\n",
       "4                10             -1         1.0        0.0      0.0       NaN\n",
       "...             ...            ...         ...        ...      ...       ...\n",
       "18493744       1003           8834         NaN        NaN      NaN       0.0\n",
       "18493745       1003           8834         NaN        NaN      NaN       0.0\n",
       "18493746       1003           8834         NaN        NaN      NaN       0.0\n",
       "18493747       1003           8834         NaN        NaN      NaN       0.0\n",
       "18493748       1003           8834         NaN        NaN      NaN       0.0\n",
       "\n",
       "[18493749 rows x 6 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_fill = data.to_pandas_df()\n",
    "df_fill"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8230d2cb-5c7d-49f8-8334-51009d32b0a7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MachineId        0\n",
       "MeasurementId    0\n",
       "PumpFailed       0\n",
       "SlowStart        0\n",
       "SlowEnd          0\n",
       "Pressure         0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_fill = df_fill.fillna(method='bfill',axis=0).fillna(0)\n",
    "df_fill.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "e97602b9-338b-4823-9606-c177df05d522",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 1., 2., 3., 4., 5., 6.])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_fill.Pressure.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "84a937cc-490b-44a2-aa9d-f819fa130bfc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MachineId</th>\n",
       "      <th>MeasurementId</th>\n",
       "      <th>PumpFailed</th>\n",
       "      <th>SlowStart</th>\n",
       "      <th>SlowEnd</th>\n",
       "      <th>Pressure</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10</td>\n",
       "      <td>-1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25451</th>\n",
       "      <td>2203</td>\n",
       "      <td>8232</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25786</th>\n",
       "      <td>1500</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25787</th>\n",
       "      <td>1501</td>\n",
       "      <td>8336</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25788</th>\n",
       "      <td>1502</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25789</th>\n",
       "      <td>1503</td>\n",
       "      <td>-1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>556 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       MachineId  MeasurementId  PumpFailed  SlowStart  SlowEnd  Pressure\n",
       "0              0              0         0.0        0.0      0.0       0.0\n",
       "1              1             -1         0.0        0.0      0.0       0.0\n",
       "2              2              0         0.0        0.0      0.0       0.0\n",
       "3              3              0         1.0        0.0      0.0       0.0\n",
       "4             10             -1         1.0        0.0      0.0       0.0\n",
       "...          ...            ...         ...        ...      ...       ...\n",
       "25451       2203           8232         0.0        0.0      0.0       0.0\n",
       "25786       1500             -1         0.0        0.0      1.0       0.0\n",
       "25787       1501           8336         0.0        0.0      1.0       0.0\n",
       "25788       1502             -1         0.0        0.0      0.0       0.0\n",
       "25789       1503             -1         0.0        0.0      0.0       0.0\n",
       "\n",
       "[556 rows x 6 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataframe = df_fill.drop_duplicates(subset='MachineId',keep='first',inplace=False)\n",
    "dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "4152f646-0ce6-4795-aed2-f2df1dc525c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataframe.Pressure.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e833a144-90e2-4df2-89db-383872df1f95",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>#                              </th><th>MachineId  </th><th>MeasurementId  </th><th>PumpFailed  </th><th>SlowStart  </th><th>SlowEnd  </th><th>Pressure  </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td><i style='opacity: 0.6'>0</i>  </td><td>0          </td><td>0              </td><td>0.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>1</i>  </td><td>1          </td><td>-1             </td><td>0.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>2</i>  </td><td>2          </td><td>0              </td><td>0.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3</i>  </td><td>3          </td><td>0              </td><td>1.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>4</i>  </td><td>10         </td><td>-1             </td><td>1.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td></tr>\n",
       "<tr><td>...                            </td><td>...        </td><td>...            </td><td>...         </td><td>...        </td><td>...      </td><td>...       </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>551</i></td><td>2203       </td><td>8232           </td><td>0.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>552</i></td><td>1500       </td><td>-1             </td><td>0.0         </td><td>0.0        </td><td>1.0      </td><td>0.0       </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>553</i></td><td>1501       </td><td>8336           </td><td>0.0         </td><td>0.0        </td><td>1.0      </td><td>0.0       </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>554</i></td><td>1502       </td><td>-1             </td><td>0.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>555</i></td><td>1503       </td><td>-1             </td><td>0.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "#    MachineId    MeasurementId    PumpFailed    SlowStart    SlowEnd    Pressure\n",
       "0    0            0                0.0           0.0          0.0        0.0\n",
       "1    1            -1               0.0           0.0          0.0        0.0\n",
       "2    2            0                0.0           0.0          0.0        0.0\n",
       "3    3            0                1.0           0.0          0.0        0.0\n",
       "4    10           -1               1.0           0.0          0.0        0.0\n",
       "...  ...          ...              ...           ...          ...        ...\n",
       "551  2203         8232             0.0           0.0          0.0        0.0\n",
       "552  1500         -1               0.0           0.0          1.0        0.0\n",
       "553  1501         8336             0.0           0.0          1.0        0.0\n",
       "554  1502         -1               0.0           0.0          0.0        0.0\n",
       "555  1503         -1               0.0           0.0          0.0        0.0"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = vaex.from_pandas(dataframe)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "05efdc00-ead1-45e6-9440-35dee06d87d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train,df_test = df.ml.train_test_split(test_size=0.3,verbose=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "92e40d30-a291-4b87-8906-ab7691f0db17",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtQAAAEOCAYAAACpYIAkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAlo0lEQVR4nO3deZhkZXn38e/PAUVZBGTEgQFHERcwEXVE8xoVxX0BYqIBN4LE0UQUE99LQaNgFMU1bvHFMaiTqCASVNS4ZSIaEwUGRGVAZBSEgQEGBdkUBe73j3Naiqa7uqarq6ur+/u5rrrqnOcsddf2nLuees5zUlVIkiRJmp67DDsASZIkaZSZUEuSJEl9MKGWJEmS+mBCLUmSJPXBhFqSJEnqgwm1JEmS1AcT6nksyVeTHDzsOOarJG9I8i/t9LIklWSzaexn2ttuwmMcl+RNg9q/JM11ncfEJH+V5LvT3M+0t92Ex1ibZJ9BPoZmlgn1HJPkho7bbUl+0zH/wk3ZV1U9o6pWDSrW8aaTGCY5Osnv2+d3bZL/TfIng4xzkjg+meR3417/v+y2TVW9var+erZi7NVElX1VvaKq3jqgx9s3yU+S3JTkW0nuO8X6ByY5P8mNSX6W5HFt+djnp/M98EeARs5M1uPt/k5LMuN1zXQSwzaW37bP5eokpyRZMtOx9RDHxeNe1xuS7NRtm9k+JvaqPf68rbOsqvasqtNmaP8vSPKLts79QpLtp1j/8CQXteufn+SBbfmzkny3PVZfkeRjSbaeiRjnAxPqOaaqthq7AZcAz+ko+/TYeoNszRyCz7bPdzHwXeCUJBlCHO/qfP2r6rNDiKGrufa+J9kBOAV4E7A9sAaY9HVL8hTgncAhwNbA44Gfj1tt2473YCA/AqRB6rUeH2GHtc/tgcC2wD8NKY7njKuzLx9SHJMadp2dZE/go8CLgR2Bm4CPdFn/r4FDgWcBWwHPBq5uF98TeBuwE/AQYCnw7kHFPmpMqEdEkn2SrE/y+iRXAJ9Isl2SLyfZmOSadnppxzZ/aNUYa4lI8p523YuSPKPL470+yWVJrk9yQZJ92/K7JDmibVn8ZZKTOn7tfqe9v7ZtLdikluaq+j2wCrgPcK/xrTLjW1Pa1sy/TXJhG+dbk+yW5HtJrmtju+u41+8NbavKxb20FCX5QJJL2/2dNdaa2i47OsmnJtnunkmOT7KhfR3flmRRu2xR+z5cneTnNBVXtxgubt+PHwE3Jtms4z24Psl5Sf6sXfchwHHAn7TvwbVt+R1aQJK8LMm6JL9KcupULTtdPBdYW1Wfq6rfAkcDD0vy4EnWfwvwj1X1/aq6raouq6rLpvnY0kjpVn8m2SLJp9rya5OcmWTHJMcAjwM+3H6nPzzBfifctl02YV00WV2xKarqV8C/Aw9tH6uSPKAjrj/UOx118OuSXNXGc0CSZyb5aVsXvaFj26OTnJzks209d3aSh03x+m6XHo+JE2z74CTfbOO4IMnzO5bdq60nr0tyBrBblxjG/mk7NMklwH+15Z9L06r76yTfSZPokmQF8ELgde378KW2/OIkT26n75bk/Ukub2/vT3K3bq9FhxcCX6qq71TVDTSNH8/NBC3LSe4CHAX8XVWdV42fte8zVfWZqvpaVd1UVdcAHwMe22Mc854J9Wi5D00r4H2BFTTv3yfa+V2B3wB3qmw7PBq4ANgBeBdwfHLnluAkDwIOAx5VVVsDTwMubhe/GjgAeALNr9RrgH9ulz2+vR9rYfxekl3bCn7XqZ5cW0H8FbC+qq6eYvUxTwceCTwGeB2wkqYC2YWmkj+oY9370Dz3nYGDgZXtc+3mTGAvmtf9M8DnkmzRQ1yrgFuABwAPB54KjFXkL6P51f9wYDnwFz3s7yCaxHvbqroF+BnNQfaeNEnqp5IsqarzgVcA32vfg23H7yjJk4B3AM8HlgC/AE7sWP7lJEf0EBPAnsAPx2aq6sY2tj0neNxFNM93cZvMr0/y4SR3H7fqL9pln0jTAi7NF93qz4Npvs+7APei+R7/pqreCPw3batwVR02wX4n3LZdNmFdNFldkaZ7wI96eTLt9/PPgR/0+PzvA2xBUwe/mSYhexFNHf444M1J7t+x/v7A57i9/v1Cks277H9Tj4ljz2NL4JvtY9ybpr79yFjSS/Me/Zamvnxpe5vKE2hacZ/Wzn8V2L3d/9nApwGqamU7PfYP6XMm2NcbaY5xewEPA/YG/qEj/muT/OkkcYyvo38G/I7m34Xxlra3h6ZpSLooyVvaRHsijwfWTrJs4akqb3P0RpPEPrmd3ofmS7BFl/X3Aq7pmD+NpuKEJlFd17HsHkAB95lgPw8ArgKeDGw+btn5wL4d80uA3wObAcvafW62Cc/x6PZ5Xds+5n8Bjxwff8dz+G7HfAGP7Zg/C3h9x/x7gfd3vH63AFt2LD8JeFM7/UmaCvPa9nb1JPFeAzysI/ZPtdN/eO40f6vdDNy9Y7uDgG+10/8FvKJj2VO7vW7t5+ClU7yO5wD7T/Q6dTy/t7XTx9NU3mPLtmrfw2XT+IweDxw7rux/gL+aYN2d2ue5pv3c7NCue0xHHMs7XsOTga8P47vnzdtM3bhjPd6t/nwp8L/AH0+wjzvUhRMsn3DbHuqiO9UVPTyf02i6DVwLXEaTDC5ulxXwgI51O+udfWgS3EXt/Nbt+o/uWP8s4IB2+mjg+x3L7gJsAB7X8brewO119hcmiHUvuh8Tv9tO/yXw3+O2/ShNa+2i9j16cMeyt0/2unH7seD+XV7Dbdt17jn+dZrkc/Mz4Jkdy54GXNzj+7WajuNNW3YZsM8E6/6fNq6vtDEuA34KvGyCdZ9Cczx84LC/Y3PlNqf6Y2pKG6v5Wx2AJPeg6bv2dGC7tnjrJIuq6tYJtr9ibKKqbmobp7cav1JVrUvyGpoKbc8kXwf+vpr+afcFPp/kto5NbqWpuKfrpKp60TS3vbJj+jcTzN+nY/6aalpQx/yCJskb856q+oeOeZK8lqZleSwZ3IYmEezmvsDmwIaOPwDuAlzaTu/UMT0Wx1Q61yfJS4C/p6nwoHkfe23N3YmmhQSAqrohyS9pWo0u7nEfY26geU06bQNcP8G6Yy1mH6qqDQBJ3kfT0vLGav6OXNOuc2WSw2hew22q6rpNjEuai7rVn/9G08J8YpJtgU/RfC9+38N+J9yWqeui6Xp1Vf3LNLb7Zcexaaw+GF9ndx6T/hBnVd2WZD13rLMPqKr/HJuZxjFxzH2BR4/r9rIZzeu6uJ2edp3d/jt3DPC8dn9j7/8OwK972NdO4x5z/LGrm+nU0e+qqmtpum9+FHgmzb8JACR5DE1r/l9U1U97jGPes8vHaKlx868FHkTzC38bbu9y0fcJfdX0lfpTmoqmaE4kg6aSeEZVbdtx26KafrDj4+vXjTQt6WPuM9mKPdqu/WtvzK7ApCexpOkv/XqarhHbVfOX6K+Z+vW9lKZVaIeO12ibqhr7+3ADzcGvM46p/OG1TTOKxsdouuXcq43r3I64pnofxn4Yje1vS5q/iafTl3ktzV+QnfvajQn+Bqymz936HuL7wyZju51GXNJcNGn9WVW/r6q3VNUeNC2FzwZe0m7X9TvTZdup6qKZrrNvYmbr7D/Uk223g6V0qbOZ/jHxUuDb496Xrarqb4CNNP9uTrvOBl5A033lyTRdc5aNi2uT6mymOHaNM76Ovj9wN5qW5/EuoPnHeNJ4kjwcOJXmX9PVPcawIJhQj7ataX5RXpvmxJajZmKnSR6U5Eltn+bfto8x9uv+OOCYNqkjyeIk+7fLNtL88r7/+H1O0zk0J0/cI82JLofOwD7fkuSubbL8bJr+eZPZmqYi3QhsluTN3PmX/p20ra/fAN6bZJs0JyLtluQJ7SonAa9OsjTJdkCv/ZXHbElT4W0ESHII7UlBrSuBpWlPyJzAZ4BDkuzVvsdvB06vqos3MQ6Az9P0t/vztm/5m4EfVdVPJln/E8Crkty7fe6vAb7cPo9Ht5+9uyS5F/BB4LSq6qUFRxoFk9afSZ6Y5I/a1szraLoZjNW7V9KlXp1s2x7qoqnqik11DvCCNCc9Pp2mH3E/HpnkuWlGyngNzY+D73dZf7rHxC8DD0zy4iSbt7dHJXlI27J9CnB0eyzag6bP+qbYuo39lzQ/ON4+bnnX9xc4AfiH9vOyA009O+EJ8RP4NPCcJI9rGzz+ETilqu7UQl1VN9GM0vS6JFunOaHzZdxeRz8U+Brwqqr6Uo+Pv2CYUI+29wN3pxnS5vs0H/SZcDfg2Ha/V9CcRDF29vUHaH6dfiPJ9e3jPhr+8GU8Bvif9iSJx6Q5KfGG9HBS4gT+iebX8pU0J9b0O9zUFTR9vi5v9/WKLokfwNdpTiT5Kc1fbL+l979KXwLcFTivfcyTafpLQtO6/HWaE0XOpqmse1ZV59H0D/8ezWvzRzR9kcf8F02rxBVJ7nRyZ9uq8Caas/M30LQoHzi2PM3FD94wfrtJYtlIc1LSMTTP89Hj9vWGJF/t2OStNCd6/pSmP+kP2m2hOaB8jeavyHNpDkCdJ5VKo27S+pOmNfdkmoT4fODb3J40fQD4izQjV3xwgv1227ZbXXSnuiLJC5NM90Szw4Hn0PRpfiHwhWnuZ8wXafo3X0Mz7Ntzp+gC836mcUxsk8un0tRdl9McK95JcyyE5t/ArdryT9I0DGyKf6U5hlxG8z6M/1FwPLBHe9z8wgTbv42mO9yPgB/THDc6R226IR0jUI17bmtpTj79NM15SlsDf9ux7XFJjuvY5DCabiKX0xxjPgN8vF32WpouK8fn9rG/PSmxlaqZ/sdHmnvSXHHqU1W1dIpVJUlDluRomhMcp3t+jTSrbKGWJEmS+mBCLUmSJPXBLh+SJElSH2yhliRJkvow0hd22WGHHWrZsmXDDkOSpuWss866uqoWDzuO2WS9LWlUdauzRzqhXrZsGWvWrJl6RUmag5L0csW1ecV6W9Ko6lZn2+VDkiRJ6oMJtSRJktQHE2pJkiSpDybUkiRJUh9MqCVJkqQ+mFBLkiRJfTChliRJkvpgQi1JkiT1wYRakiRJ6sNIXylxrlp2xFeGHcJIufjYZw07BEkLnPX2prHelu7IFmpJkiSpDybUkiRJUh9MqCVJACT5eJKrkpzbUbZ9km8mubC9365j2ZFJ1iW5IMnThhO1JA2fCbUkacwngaePKzsCWF1VuwOr23mS7AEcCOzZbvORJItmL1RJmjtMqCVJAFTVd4BfjSveH1jVTq8CDugoP7Gqbq6qi4B1wN6zEackzTUm1JKkbnasqg0A7f292/KdgUs71lvflt1JkhVJ1iRZs3HjxoEGK0nDYEItSZqOTFBWE61YVSuranlVLV+8ePGAw5Kk2WdCLUnq5sokSwDa+6va8vXALh3rLQUun+XYJGlOMKGWJHVzKnBwO30w8MWO8gOT3C3J/YDdgTOGEJ8kDZ1XSpQkAZDkBGAfYIck64GjgGOBk5IcClwCPA+gqtYmOQk4D7gFeGVV3TqUwCVpyEyoJUkAVNVBkyzad5L1jwGOGVxEkjQa7PIhSZIk9cGEWpIkSeqDCbUkSZLUBxNqSZIkqQ8m1JIkSVIfTKglSZKkPphQS5IkSX0woZYkSZL6YEItSZIk9cGEWpIkSeqDCbUkSZLUBxNqSZIkqQ8m1JIkSVIfTKglSZKkPphQS5IkSX0woZYkSZL6MLCEOskuSb6V5Pwka5Mc3pZvn+SbSS5s77fr2ObIJOuSXJDkaYOKTZIkSZopg2yhvgV4bVU9BHgM8MokewBHAKurandgdTtPu+xAYE/g6cBHkiwaYHySJElS3waWUFfVhqo6u52+Hjgf2BnYH1jVrrYKOKCd3h84sapurqqLgHXA3oOKT5IkSZoJs9KHOsky4OHA6cCOVbUBmqQbuHe72s7ApR2brW/Lxu9rRZI1SdZs3LhxoHFLkiRJUxl4Qp1kK+DfgddU1XXdVp2grO5UULWyqpZX1fLFixfPVJiSJEnStAw0oU6yOU0y/emqOqUtvjLJknb5EuCqtnw9sEvH5kuBywcZnyRJktSvQY7yEeB44Pyqel/HolOBg9vpg4EvdpQfmORuSe4H7A6cMaj4JEmSpJmw2QD3/VjgxcCPk5zTlr0BOBY4KcmhwCXA8wCqam2Sk4DzaEYIeWVV3TrA+CRJkqS+DSyhrqrvMnG/aIB9J9nmGOCYQcUkSZIkzTSvlChJkiT1wYRakiRJ6oMJtSRJktQHE2pJkiSpD1Mm1En+rZcySZIkaSHqpYV6z86ZJIuARw4mHEmSJGm0TJpQJzkyyfXAHye5Lsn17fxV3H4xFknSPJfk75KsTXJukhOSbJFk+yTfTHJhe7/dsOOUpGGZNKGuqndU1dbAu6tqm6raur3dq6qOnMUYJUlDkmRn4NXA8qp6KLAIOBA4AlhdVbsDq9t5SVqQpuzyUVVHJtkvyXva27NnIzBJ0pyxGXD3JJsB9wAuB/YHVrXLVwEHDCc0SRq+Xk5KfAdwOM0lwc8DDm/LJEnzXFVdBrwHuATYAPy6qr4B7FhVG9p1NgD3nmwfSVYkWZNkzcaNG2cjbEmaVb2clPgs4ClV9fGq+jjw9LZMkjTPtX2j9wfuB+wEbJnkRZuyj6paWVXLq2r54sWLBxGmJA1Vr+NQb9sxfc8BxCFJmpueDFxUVRur6vfAKcD/Aa5MsgSgvb9qiDFK0lBt1sM67wB+kORbQIDHA56UKEkLwyXAY5LcA/gNsC+wBrgROBg4tr139CdJC9aUCXVVnZDkNOBRNAn166vqikEHJkkavqo6PcnJwNnALcAPgJXAVsBJSQ6lSbqfN7woJWm4emmhhqZryNXt+g9M8sCq+s7gwpIkzRVVdRRw1Ljim2laqyVpwZsyoU7yTuAvgbXAbW1xASbUkiRJWvB6aaE+AHhQVd084FgkSZKkkdPLKB8/BzYfdCCSJEnSKOqlhfom4Jwkq2n6zAFQVa8eWFSSJEnSiOgloT61vUmSJEkap5dh81bNRiCSJEnSKOr1SomSJEmSJmBCLUmSJPWha0KdZFGSd89WMJIkSdKo6ZpQV9WtwCOTZJbikSRJkkZKL6N8/AD4YpLPATeOFVbVKQOLSpIkSRoRvSTU2wO/BJ7UUVaACbUkSZIWvF6GzTtkNgKRJEmSRtGUo3wkWZrk80muSnJlkn9PsnQ2gpMkSZLmul6GzfsEzZUSdwJ2Br7UlkmSJEkLXi8J9eKq+kRV3dLePgksHnBckiRJ0kjoJaG+OsmL2jGpFyV5Ec1JipIkSdKC10tC/VLg+cAVwAbgL9oySZIkacGbdJSPJAcB36iqS4D9Zi8kSZIkaXR0GzbvvsDnkmwOrAa+CpxRVTUrkUmSJEkjYNIuH1V1bFU9CXgm8EOabh5nJ/lMkpck2bHbjpN8vB1q79yOsqOTXJbknPb2zI5lRyZZl+SCJE/r/6lJkiRJg9fLhV2uBz7f3kiyB/AM4F+BbonvJ4EPt+t1+qeqek9nQbvPA4E9aYbn+88kD6yqW3t7GtLCsuyIrww7hJFy8bHPGnYIkqR5rFsf6kd02e5bVfXebjuuqu8kWdZjHPsDJ1bVzcBFSdYBewPf63F7SZIkaSi6tVCPJcxbAMtpun0E+GPgdOBPp/mYhyV5CbAGeG1VXUNzwZjvd6yzvi27kyQrgBUAu+666zRDkCRJkmZGtz7UT6yqJwK/AB5RVcur6pHAw4F103y8/wfsBuxFMwTfWNKeiUKYJK6VbSzLFy/2+jKSJEkarl7GoX5wVf14bKaqzqVJiDdZVV1ZVbdW1W3Ax2i6dUDTIr1Lx6pLgcun8xiSJEnSbJrypETg/CT/AnyKptX4RcD503mwJEuqakM7+2fA2AggpwKfSfI+mpMSdwfOmM5jSJIkzSWeSL5pRvFE8l4S6kOAvwEOb+e/Q9N1o6skJwD7ADskWQ8cBeyTZC+axPxi4OUAVbU2yUnAecAtwCsd4UOSJEmjoJdh836b5DjgP6rqgl53XFUHTVB8fJf1jwGO6XX/kiRJ0lwwZR/qJPsB5wBfa+f3SnLqgOOSJEmSRkIvJyUeRXPy4LUAVXUOsGxgEUmSJEkjpJeE+paq+vXAI5EkzVlJtk1ycpKfJDk/yZ8k2T7JN5Nc2N5vN+w4JWkYekmoz03yAmBRkt2TfAj43wHHJUmaWz4AfK2qHgw8jGa0pyOA1VW1O7C6nZekBaeXhPpVwJ7AzcAJwHXAawYYkyRpDkmyDfB42hPLq+p3VXUtsD+wql1tFXDAMOKTpGHrZZSPm4A3tjdJ0sJzf2Aj8IkkDwPOohlKdcexawtU1YYk955o4yQrgBUAu+666+xELEmzaNKEOsmXmOTy3wBVtd9AIpIkzTWbAY8AXlVVpyf5AJvQvaOqVgIrAZYvXz7pcUWSRlW3Fur3zFoUkqS5bD2wvqpOb+dPpkmorxy7Am6SJcBVQ4tQkoZo0oS6qr49m4FIkuamqroiyaVJHtRe4GtfmivbngccDBzb3n9xiGFK0tB06/JxUlU9P8mPmaDrR1X98UAjkyTNJa8CPp3krsDPgUNoTmw/KcmhwCXA84YYnyQNTbcuH4e398+ejUAkSXNXe1Gv5RMs2neWQ5GkOadbl4+xM7d/MXvhSJIkSaNlynGokzwmyZlJbkjyuyS3JrluNoKTJEmS5rpeLuzyYeAg4ELg7sBfAx8aZFCSJEnSqJjywi4AVbUuyaKqupVmYH8vPS5JkiTRW0J9U3tW9zlJ3gVsALYcbFiSJEnSaJi0y0eSsWT7xe16hwE3ArsAfz740CRJkqS5r1sL9RnAI6rqF0k+VFWvAt4yS3FJkiRJI6HbSYnpmH7soAORJEmSRlG3hPpOV0eUJEmSdEfdunw8OMmPaFqqd2unaefLS49LkiRJ3RPqh8xaFJIkSdKI6nbpcS85LkmSJE2hlyslSpIkSZqECbUkSZLUh24Xdlnd3r9z9sKRJEmSRku3kxKXJHkCsF+SE7njuNRU1dkDjUySJEkaAd0S6jcDRwBLgfeNW1bAkwYVlCRJkjQquo3ycTJwcpI3VdVbZzEmSZIkaWR0a6EGoKremmQ/4PFt0WlV9eXBhiVJkiSNhilH+UjyDuBw4Lz2dnhbJkmSJC14U7ZQA88C9qqq2wCSrAJ+ABw5yMAkSZKkUdDrONTbdkzfcwBxSJIkSSOplxbqdwA/SPItmqHzHo+t05IkSRLQ20mJJyQ5DXgUTUL9+qq6YtCBSZIkSaOgpy4fVbWhqk6tqi/2mkwn+XiSq5Kc21G2fZJvJrmwvd+uY9mRSdYluSDJ0zb9qUiSJEmzr9c+1NPxSeDp48qOAFZX1e7A6naeJHsABwJ7ttt8JMmiAcYmSZIkzYiBJdRV9R3gV+OK9wdWtdOrgAM6yk+sqpur6iJgHbD3oGKTJEmSZkrXhDrJXTq7bMyAHatqAzTdSIB7t+U7A5d2rLe+LZsophVJ1iRZs3HjxhkMTZIkSdp0XRPqduzpHybZdcBxZKKHn2jFqlpZVcuravnixYsHHJYkSZLUXS/D5i0B1iY5A7hxrLCq9pvG412ZZElVbUiyBLiqLV8P7NKx3lLg8mnsX5IkSZpVvSTUb5nBxzsVOBg4tr3/Ykf5Z5K8D9gJ2B04YwYfV5IkSRqIXsah/naS+wK7V9V/JrkHMOUIHElOAPYBdkiyHjiKJpE+KcmhwCXA89rHWJvkJOA84BbglVV16zSfkyRphrUjL60BLquqZyfZHvgssAy4GHh+VV0zvAglaXimTKiTvAxYAWwP7EZzsuBxwL7dtquqgyZZNOF2VXUMcMxU8UiShuJw4Hxgm3Z+bBjUY5Mc0c6/fljBSdIw9TJs3iuBxwLXAVTVhdw+OockaZ5LshR4FvAvHcWTDYMqSQtOLwn1zVX1u7GZJJsxyQgckqR56f3A64DbOsomGwb1ThzuVNJ810tC/e0kbwDunuQpwOeALw02LEnSXJDk2cBVVXXWdPfhcKeS5rteRvk4AjgU+DHwcuA/uOPffpKk+euxwH5JnglsAWyT5FNMPgyqJC04vYzycVuSVcDpNF09Lqgqu3xI0gJQVUcCRwIk2Qf4v1X1oiTvZuJhUCVpwelllI9n0Yzq8TOaKxreL8nLq+qrgw5OkjRnTTgMqiQtRL10+Xgv8MSqWgeQZDfgK4AJtSQtIFV1GnBaO/1Lphg+VZIWil5OSrxqLJlu/Rz7ykmSJElAlxbqJM9tJ9cm+Q/gJJo+1M8DzpyF2CRJkqQ5r1uXj+d0TF8JPKGd3ghsN7CIJEmSpBEyaUJdVYfMZiCSJEnSKOpllI/7Aa8ClnWuX1X7DS4sSZIkaTT0MsrHF4Djaa6OeFv3VSVJkqSFpZeE+rdV9cGBRyJJkiSNoF4S6g8kOQr4BnDzWGFVnT2wqCRJkqQR0UtC/UfAi4EncXuXj2rnJUmSpAWtl4T6z4D7V9XvBh2MJEmSNGp6uVLiD4FtBxyHJEmSNJJ6aaHeEfhJkjO5Yx9qh82TJEnSgtdLQn3UwKOQJEmSRtSUCXVVfXs2ApEkSZJGUS9XSryeZlQPgLsCmwM3VtU2gwxMkiRJGgW9tFBv3Tmf5ABg70EFJEmSJI2SXkb5uIOq+gKOQS1JkiQBvXX5eG7H7F2A5dzeBUSSJEla0HoZ5eM5HdO3ABcD+w8kGkmSJGnE9NKH+pDZCESSJEkaRZMm1Ene3GW7qqq3DiAeSZIkaaR0a6G+cYKyLYFDgXsBJtSSJEla8CZNqKvqvWPTSbYGDgcOAU4E3jvZdpIkSdJC0rUPdZLtgb8HXgisAh5RVdfMRmCSJEnSKOjWh/rdwHOBlcAfVdUNsxaVJEmSNCK6XdjltcBOwD8Alye5rr1dn+S62QlPkiRJmtu69aHe5KsoSpIkSQuNSbMkSZLUh16ulDjjklwMXA/cCtxSVcvbEyA/CyyjuRrj8z0BUpIkSXPdMFuon1hVe1XV8nb+CGB1Ve0OrG7nJUmSpDltLnX52J9maD7a+wOGF4okSZLUm2El1AV8I8lZSVa0ZTtW1QaA9v7eE22YZEWSNUnWbNy4cZbClaSFK8kuSb6V5Pwka5Mc3pZvn+SbSS5s77cbdqySNAzDSqgfW1WPAJ4BvDLJ43vdsKpWVtXyqlq+ePHiwUUoSRpzC/DaqnoI8BiaensP7KonScCQEuqqury9vwr4PLA3cGWSJQDt/VXDiE2SdEdVtaGqzm6nrwfOB3bGrnqSBAwhoU6yZZKtx6aBpwLnAqcCB7erHQx8cbZjkyR1l2QZ8HDgdHrsqidJ890whs3bEfh8krHH/0xVfS3JmcBJSQ4FLgGeN4TYJEmTSLIV8O/Aa6rqurYe72W7FcAKgF133XVwAUrSkMx6Ql1VPwceNkH5L4F9ZzseSdLUkmxOk0x/uqpOaYuvTLKkqjZ066pXVSuBlQDLly+vWQlYkmbRXBo2T5I0B6Vpij4eOL+q3texyK56ksSQrpQoSRopjwVeDPw4yTlt2RuAY7GrniSZUEuSuquq7wKTdZi2q56kBc8uH5IkSVIfTKglSZKkPphQS5IkSX0woZYkSZL6YEItSZIk9cGEWpIkSeqDCbUkSZLUBxNqSZIkqQ8m1JIkSVIfTKglSZKkPphQS5IkSX0woZYkSZL6YEItSZIk9cGEWpIkSeqDCbUkSZLUBxNqSZIkqQ8m1JIkSVIfTKglSZKkPphQS5IkSX0woZYkSZL6YEItSZIk9cGEWpIkSeqDCbUkSZLUBxNqSZIkqQ8m1JIkSVIfTKglSZKkPphQS5IkSX0woZYkSZL6YEItSZIk9cGEWpIkSeqDCbUkSZLUBxNqSZIkqQ9zLqFO8vQkFyRZl+SIYccjSZqcdbYkzbGEOski4J+BZwB7AAcl2WO4UUmSJmKdLUmNOZVQA3sD66rq51X1O+BEYP8hxyRJmph1tiQBmw07gHF2Bi7tmF8PPLpzhSQrgBXt7A1JLpil2OaDHYCrhx3EeHnnsCPQDPIztmnuO+wA+jRlnQ3W233yO6VB8vO1aSats+daQp0JyuoOM1UrgZWzE878kmRNVS0fdhyav/yMLThT1tlgvd0Pv1MaJD9fM2eudflYD+zSMb8UuHxIsUiSurPOliTmXkJ9JrB7kvsluStwIHDqkGOSJE3MOluSmGNdPqrqliSHAV8HFgEfr6q1Qw5rPvEvVw2an7EFxDp7Vvid0iD5+ZohqbpTdzdJkiRJPZprXT4kSZKkkWJCLUmSJPXBhFqSJEnqgwm1JEmS1Ic5NcqHZl6SHWmuZlbA5VV15ZBDkiRNwjpbGk2O8jFPJdkLOA64J3BZW7wUuBb426o6eziRab4xAZD6Z52t2WKdPRgm1PNUknOAl1fV6ePKHwN8tKoeNpTANG+YAEgzxzpbg2adPVgm1PNUkguravdJlq2rqgfMdkyaX0wApJljna1Bs84eLPtQz19fTfIV4F+BS9uyXYCXAF8bWlSaT7YcXzEDVNX3k2w5jICkEWadrUGzzh4gW6jnsSTPAPan6SsVYD1walX9x1AD07yQ5IPAbkycAFxUVYcNKzZpFFlna5CsswfLhFrStJkASNLosM4eHBPqBSjJiqpaOew4JElTs86W5j4v7LIwZdgBaH5LsmLYMUjziHW2Bso6u3+elDiPJXkwt/+1U8DlNH/tfHSogWkhMAGQNpF1tobIOrtPtlDPU0leD5xI8yU5AziznT4hyRHDjE0Lwu+GHYA0SqyzNWTW2X2yD/U8leSnwJ5V9ftx5XcF1k423qk0E5JcUlW7DjsOaVRYZ2uYrLP7Z5eP+es2YCfgF+PKl7TLpL4k+dFki4AdZzMWaR6wztZAWWcPlgn1/PUaYHWSC7l9vMldgQcAjjWpmbAj8DTgmnHlAf539sORRtprsM7WYFlnD5AJ9TxVVV9L8kBgb+443uSZVXXrUIPTfPFlYKuqOmf8giSnzXo00gizztYssM4eIPtQS5IkSX1wlA9JkiSpDybUkiRJUh9MqCVJkqQ+mFBLkiRJffj/whL9LuXdSqsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Inspect the target variable \n",
    "train_pumpfailed_value_counts = df_train.PumpFailed.value_counts()\n",
    "test_pumpfailed_value_counts = df_test.PumpFailed.value_counts()\n",
    "\n",
    "plt.figure(figsize=(12,4))\n",
    "plt.subplot(121)\n",
    "\n",
    "train_pumpfailed_value_counts.plot.bar()\n",
    "train_pumpfailed_ratio = train_pumpfailed_value_counts[1]/train_pumpfailed_value_counts[0]\n",
    "plt.title(f'Train set: PumpFailed ratio: {train_pumpfailed_ratio: .2f}')\n",
    "plt.ylabel('Number of Failed/or not')\n",
    "\n",
    "plt.subplot(122)\n",
    "test_pumpfailed_value_counts.plot.bar()\n",
    "test_pumpfailed_ratio = test_pumpfailed_value_counts[1]/test_pumpfailed_value_counts[0]\n",
    "plt.title(f'Test set: PumpFailed ratio: {test_pumpfailed_ratio:.2f}')\n",
    "\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "f39f99d6-0985-4f2d-9d4a-3f80bace8db8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: xgboost in /opt/conda/lib/python3.7/site-packages (1.5.2)\n",
      "Requirement already satisfied: scipy in /opt/conda/lib/python3.7/site-packages (from xgboost) (1.7.3)\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from xgboost) (1.19.5)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install xgboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f0529bb1-4e7f-4e4b-a17d-813a79ff87a9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>#                              </th><th>MachineId  </th><th>MeasurementId  </th><th>PumpFailed  </th><th>SlowStart  </th><th>SlowEnd  </th><th>Pressure  </th><th>prediction_xgb  </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td><i style='opacity: 0.6'>0</i>  </td><td>2113       </td><td>1312           </td><td>0.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>1</i>  </td><td>2120       </td><td>1319           </td><td>1.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>1               </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>2</i>  </td><td>2121       </td><td>1319           </td><td>1.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>1               </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3</i>  </td><td>2122       </td><td>1319           </td><td>1.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>1               </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>4</i>  </td><td>2123       </td><td>-1             </td><td>0.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td></tr>\n",
       "<tr><td>...                            </td><td>...        </td><td>...            </td><td>...         </td><td>...        </td><td>...      </td><td>...       </td><td>...             </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>384</i></td><td>2203       </td><td>8232           </td><td>0.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>385</i></td><td>1500       </td><td>-1             </td><td>0.0         </td><td>0.0        </td><td>1.0      </td><td>0.0       </td><td>0               </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>386</i></td><td>1501       </td><td>8336           </td><td>0.0         </td><td>0.0        </td><td>1.0      </td><td>0.0       </td><td>0               </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>387</i></td><td>1502       </td><td>-1             </td><td>0.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>388</i></td><td>1503       </td><td>-1             </td><td>0.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "#    MachineId    MeasurementId    PumpFailed    SlowStart    SlowEnd    Pressure    prediction_xgb\n",
       "0    2113         1312             0.0           0.0          0.0        0.0         0\n",
       "1    2120         1319             1.0           0.0          0.0        0.0         1\n",
       "2    2121         1319             1.0           0.0          0.0        0.0         1\n",
       "3    2122         1319             1.0           0.0          0.0        0.0         1\n",
       "4    2123         -1               0.0           0.0          0.0        0.0         0\n",
       "...  ...          ...              ...           ...          ...        ...         ...\n",
       "384  2203         8232             0.0           0.0          0.0        0.0         0\n",
       "385  1500         -1               0.0           0.0          1.0        0.0         0\n",
       "386  1501         8336             0.0           0.0          1.0        0.0         0\n",
       "387  1502         -1               0.0           0.0          0.0        0.0         0\n",
       "388  1503         -1               0.0           0.0          0.0        0.0         0"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import xgboost\n",
    "import vaex.ml.sklearn\n",
    "\n",
    "features = ['MachineId','MeasurementId','SlowStart','SlowEnd','Pressure']\n",
    "target = 'PumpFailed'\n",
    "\n",
    "xgb_model = xgboost.sklearn.XGBClassifier(max_depth=11,\n",
    "                                          learning_rate=0.1,\n",
    "                                          n_estimators=500,\n",
    "                                          subsample=0.75,\n",
    "                                          colsample_bylevel=1,\n",
    "                                          colsample_bytree=1,\n",
    "                                          scale_pos_weight=1.5,\n",
    "                                          reg_lambda=1.5,\n",
    "                                          reg_alpha=5,\n",
    "                                          n_jobs=-1,\n",
    "                                          random_state=42,\n",
    "                                          verbosity=0,\n",
    "                                          use_label_encoder=False)\n",
    "\n",
    "vaex_xgb_model = vaex.ml.sklearn.Predictor(features=features,\n",
    "                                           target='PumpFailed',\n",
    "                                           model=xgb_model,\n",
    "                                           prediction_name='prediction_xgb')\n",
    "\n",
    "\n",
    "#Train the model\n",
    "vaex_xgb_model.fit(df_train)\n",
    "#Get the prediction of the model on the training data\n",
    "df_train = vaex_xgb_model.transform(df_train)\n",
    "\n",
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c5d62786-c778-41bc-b25c-42044b0ec690",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, f1_score, roc_auc_score\n",
    "def binary_metrics(y_true, y_pred):\n",
    "    acc = accuracy_score(y_true=y_true, y_pred=y_pred)\n",
    "    f1 = f1_score(y_true=y_true, y_pred=y_pred)\n",
    "    roc = roc_auc_score(y_true=y_true, y_score=y_pred)\n",
    "    print(f'Accuracy:{acc: .3f}')\n",
    "    print(f'f1 score: {f1:.3f}')\n",
    "    print(f'roc_auc: {roc: .3f}')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "0e823648-a115-443e-b761-ad19c6da2083",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metrics for the training set:\n",
      "Accuracy: 0.887\n",
      "f1 score: 0.867\n",
      "roc_auc:  0.895\n"
     ]
    }
   ],
   "source": [
    "print('Metrics for the training set:')\n",
    "binary_metrics(y_true=df_train.PumpFailed.values, y_pred=df_train.prediction_xgb.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "dbe8cfd2-29db-44bc-b9a1-6e70c8f71f74",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>#                             </th><th>MachineId  </th><th>MeasurementId  </th><th>PumpFailed  </th><th>SlowStart  </th><th>SlowEnd  </th><th>Pressure  </th><th>prediction_xgb  </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td><i style='opacity: 0.6'>0</i> </td><td>33         </td><td>127            </td><td>0.0         </td><td>0.0        </td><td>1.0      </td><td>0.0       </td><td>0               </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>1</i> </td><td>1000       </td><td>-1             </td><td>1.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>2</i> </td><td>1001       </td><td>-1             </td><td>1.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3</i> </td><td>1002       </td><td>-1             </td><td>1.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>4</i> </td><td>1003       </td><td>-1             </td><td>1.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td></tr>\n",
       "<tr><td>...                           </td><td>...        </td><td>...            </td><td>...         </td><td>...        </td><td>...      </td><td>...       </td><td>...             </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>95</i></td><td>2102       </td><td>-1             </td><td>1.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>96</i></td><td>2103       </td><td>1305           </td><td>1.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>97</i></td><td>2110       </td><td>-1             </td><td>0.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>98</i></td><td>2111       </td><td>-1             </td><td>0.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>99</i></td><td>2112       </td><td>-1             </td><td>0.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "#    MachineId    MeasurementId    PumpFailed    SlowStart    SlowEnd    Pressure    prediction_xgb\n",
       "0    33           127              0.0           0.0          1.0        0.0         0\n",
       "1    1000         -1               1.0           0.0          0.0        0.0         0\n",
       "2    1001         -1               1.0           0.0          0.0        0.0         0\n",
       "3    1002         -1               1.0           0.0          0.0        0.0         0\n",
       "4    1003         -1               1.0           0.0          0.0        0.0         0\n",
       "...  ...          ...              ...           ...          ...        ...         ...\n",
       "95   2102         -1               1.0           0.0          0.0        0.0         0\n",
       "96   2103         1305             1.0           0.0          0.0        0.0         0\n",
       "97   2110         -1               0.0           0.0          0.0        0.0         0\n",
       "98   2111         -1               0.0           0.0          0.0        0.0         0\n",
       "99   2112         -1               0.0           0.0          0.0        0.0         0"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state = df_train.state_get()\n",
    "df_test.state_set(state)\n",
    "df_test.tail(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "e4b74ffd-5d53-4ad0-83e2-e84a5f684a64",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metrics for the test set:\n",
      "Accuracy: 0.497\n",
      "f1 score: 0.333\n",
      "roc_auc:  0.465\n"
     ]
    }
   ],
   "source": [
    "print('Metrics for the test set:')\n",
    "binary_metrics(y_true=df_test.PumpFailed.values, y_pred=df_test.prediction_xgb.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "2b3d59bd-2fd9-4c8f-bc2b-88a69c714bd8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbEAAAIYCAYAAADjK9IKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAb+0lEQVR4nO3de7SddX3n8fcHIniBAcd4SVGMxaijghECFiqIVp0qLgFlLMhSvIxU7TAdR2xxOXVp7UWL460OOlSRylK0KiKKis6CGBREgoREURyLeAGroGOUS1MJ3/njPFnudUxO9klOzj7f8H6tlXX2fp5nP/v3O1vzzm/v5xxSVUiS1NEukx6AJEnbyohJktoyYpKktoyYJKktIyZJasuISZLaMmKSfkuSfZPcmmTXSY9FmokRk3ZiSY5PckWS25L8dLj9yiSZ6XFV9YOq2qOqNs7XWKVtYcSknVSSVwPvBE4HHgQ8EHg58PvAbhMcmjRn4m/skHY+SfYCbgJeWFWf2MIxRwF/BewHrAfeX1VvGPYtBb4H3KOq7kyyErgUeApwAHA58PyqumXHzkSamSsxaed0KLA78KkZjrkNeCGwN3AU8Iokx8xw/POBFwMPYGold+pcDFTaHkZM2jktBm6pqjs3bUhyWZJfJLkjyRFVtbKq1lXVXVW1FjgXeNIM5/xAVX2nqu4A/glYvkNnII3BiEk7p58Bi5Ms2rShqg6rqr2HfbskeUKSS5LcnGQ9U5+XLZ7hnP8ycvt2YI8dMG5pVoyYtHO6HNgAHD3DMR8GLgAeUlV7Ae8FZrxqUVpojJi0E6qqXwBvBM5IclySPZLskmQ5cJ/hsD2Bn1fVvyY5hKnPvKRWFm39EEkdVdXfJbkR+DPgg0xdyHE98OfAZcArgf+Z5N3Al5j6nGvvyYxW2jZeYi9Jasu3EyVJbRkxSVJbRkyS1JYRkyS1ZcQkSW15if08W7x4cS1dunTSw5CkVq666qpbqur+07cbsXm2dOlSVq9ePelhSFIrSb6/ue2+nShJasuISZLaMmKSpLaMmCSpLSMmSWrLiEmS2jJikqS2jJgkqS0jJklqy4hJktoyYpKktoyYJKktIyZJasuISZLaMmKSpLaMmCSpLSMmSWrLiEmS2jJikqS2jJgkqa1Fkx7A3c26G9ez9LQLJz0MSZpXN7z5qB1yXldikqS2jJgkqS0jJklqy4hJktoyYpKktoyYJKmtdhFL8rok30yyNsmaJE9IsjLJiu0878ok1w3nXJPk47N8/NlJjtueMUiSZqfVz4klORR4FnBgVW1IshjYbQ6f4sSqWj2H55Mk7UDdVmJLgFuqagNAVd1SVTeNHpDkhCTrknwjyVuGbc9L8rbh9p8muX64vV+SL8/0hMMK611JLkty/abVVqa8O8m1SS4EHjD305UkzaRbxL4APCTJd5KckeRJozuT/A7wFuApwHLg4CTHAKuAw4fDDgd+lmQf4InApSOn+NDI24mnj2xfMhz7LODNw7ZjgUcC+wMvAw7b0qCTnJxkdZLVG29fvw3TliRtTqu3E6vq1iQHMRWiJwMfTXLayCEHAyur6maAJB8Cjqiq85PskWRP4CHAh4EjhvOcN/L4Lb2deH5V3QVcm+SBw7YjgHOraiNwU5KLZxj3mcCZALsvWVazn7kkaXNaRQxgiMZKYGWSdcBJI7szw0MvB14MXMfU6uslwKHAq8d42g1beA6DJEkT1OrtxCSPTLJsZNNy4Psj968AnpRkcZJdgROALw37VgGnDl+vZmolt6GqtvX9vVXA8Ul2TbJkOJ8kaR51W4ntAfx9kr2BO4HvAicDHweoqh8neS1wCVMrps9W1aeGx17K1FuJq6pqY5IfAt+edv4PJbljuH1LVT11hrF8kqnP3tYB3+E3sZQkzZNU+Y7YfNp9ybJactI7Jj0MSZpX2/ufYklyVVX91s8Dt3o7UZKkUUZMktSWEZMktWXEJEltGTFJUlvdLrFvb/999mL1dl6lI0ma4kpMktSWEZMktWXEJEltGTFJUltGTJLUlhGTJLVlxCRJbRkxSVJbRkyS1JYRkyS1ZcQkSW0ZMUlSW0ZMktSWEZMktWXEJEltGTFJUltGTJLUlhGTJLVlxCRJbRkxSVJbRkyS1JYRkyS1ZcQkSW0ZMUlSW0ZMktSWEZMktWXEJEltGTFJUltGTJLUlhGTJLVlxCRJbRkxSVJbRkyS1JYRkyS1ZcQkSW0ZMUlSW0ZMktSWEZMktWXEJEltGTFJUltGTJLUlhGTJLVlxCRJbRkxSVJbRkyS1JYRkyS1ZcQkSW0ZMUlSW0ZMktSWEZMktWXEJEltGTFJUltGTJLUlhGTJLVlxCRJbRkxSVJbRkyS1JYRkyS1ZcQkSW0ZMUlSW0ZMktSWEZMktWXEJEltGTFJUltGTJLUlhGTJLVlxCRJbRkxSVJbRkyS1JYRkyS1ZcQkSW0ZMUlSW0ZMktSWEZMktWXEJEltGTFJUltGTJLUlhGTJLVlxCRJbRkxSVJbRkyS1JYRkyS1ZcQkSW0ZMUlSW4smPYC7m3U3rmfpaRdOehga3PDmoyY9BEnbwZWYJKktIyZJasuISZLaMmKSpLaMmCSpLSMmSWprQUcsyeuSfDPJ2iRrkjwhycokK7bzvM9KcnWSa5Jcm+SPh+3HJHn0NpzvyCSHbc+YJEmzt2B/TizJocCzgAOrakOSxcBuc3DeewBnAodU1Y+S7A4sHXYfA3wGuHYW51sEHAncCly2veOTJI1vIa/ElgC3VNUGgKq6papuGj0gyQlJ1iX5RpK3DNuel+Rtw+0/TXL9cHu/JF8G9mQq3j8bzruhqq4bVlLPBk4fVn37JXlZkiuHFdsnktx7ONfZSd6W5BLgo8DLgVcNjzt8Hr43kiQWdsS+ADwkyXeSnJHkSaM7k/wO8BbgKcBy4OAkxwCrgE0hORz4WZJ9gCcCl1bVz4ELgO8nOTfJiUl2qarLhu2vqarlVfXPwHlVdXBVPQ74FvDSkSE8AnhqVT0XeC/w9uFxl06fSJKTk6xOsnrj7evn5rsjSVq4EauqW4GDgJOBm4GPJnnRyCEHAyur6uaquhP4EHBEVf0LsEeSPYGHAB8GjmAqaJcO5/7PwB8AXwNOBc7awjAem+TSJOuAE4HHjOz7WFVtHHMuZ1bViqpaseu99xrnIZKkMSzYz8QAhkisBFYOITlpZHdmeOjlwIuB65gK10uAQ4FXj5x7HbAuyTnA94AXbeY8ZwPHVNU1Q0CPHNl326wmI0macwt2JZbkkUmWjWxaDnx/5P4VwJOSLE6yK3AC8KVh3yqmVlirgKuBJwMbqmp9kj2SHLmF8/6Kqc/MNtkT+PFwMciJMwx3+uMkSfNgwUYM2AP4x+ES+LXAo4E3bNpZVT8GXgtcAlwDfL2qPjXsvpSptxJXDau5HwJfHvYF+LMk1yVZA7yR36zCPgK8Zrj8fj/gL5iK5ReBb88w1k8Dx3phhyTNr1TVpMdwt7L7kmW15KR3THoYGvifYpF6SHJVVf3Wzwgv5JWYJEkzMmKSpLaMmCSpLSMmSWrLiEmS2lrQP+y8M9p/n71Y7RVxkjQnXIlJktoyYpKktoyYJKktIyZJasuISZLaMmKSpLaMmCSpLSMmSWrLiEmS2jJikqS2jJgkqS0jJklqy4hJktoyYpKktoyYJKktIyZJasuISZLaMmKSpLaMmCSpLSMmSWrLiEmS2jJikqS2jJgkqS0jJklqy4hJktoyYpKktoyYJKktIyZJasuISZLaMmKSpLaMmCSpLSMmSWrLiEmS2jJikqS2jJgkqS0jJklqy4hJktoyYpKktoyYJKktIyZJasuISZLaMmKSpLaMmCSpLSMmSWrLiEmS2jJikqS2jJgkqS0jJklqy4hJktoyYpKktoyYJKktIyZJasuISZLaMmKSpLaMmCSpLSMmSWrLiEmS2jJikqS2jJgkqS0jJklqy4hJktoyYpKktoyYJKktIyZJasuISZLaMmKSpLaMmCSpLSMmSWrLiEmS2jJikqS2jJgkqS0jJklqy4hJktoyYpKktoyYJKktIyZJasuISZLaMmKSpLaMmCSpLSMmSWrLiEmS2jJikqS2jJgkqS0jJklqa9GkB3B3s+7G9Sw97cJJD2OibnjzUZMegqSdhCsxSVJbRkyS1JYRkyS1ZcQkSW0ZMUlSW0ZMktTWViOWpJKcM3J/UZKbk3xmxw5tYUhyZJLDRu6/IcmpmzluaZJvzO/oJOnubZyV2G3AY5Pca7j/NODGHTekLUsyiZ9rOxI4bGsHSZLm37hvJ34O2PQTqicA527akeQ+Sc5KcmWSq5McPWxfmuTSJF8f/hw2bF+SZFWSNUm+keTwYfutI+c8LsnZw+2zk7wtySXAW5Lsl+TzSa4azv+okePek+SSJNcnedIwrm9tOtdw3NOTXD6M6WNJ9hi235DkjcP2dUkelWQp8HLgVcN4Dx/9piQ5KMk1SS4H/mTM76UkaY6MG7GPAMcnuSdwAHDFyL7XARdX1cHAk4HTk9wH+CnwtKo6EPgj4F3D8c8HLqqq5cDjgDVjPP8jgKdW1auBM4FTquog4FTgjJHj7gs8BXgV8Gng7cBjgP2TLE+yGPgfw7kOBFYD/33k8bcM298DnFpVNwDvBd5eVcur6tJp4/oA8F+r6tCZBp/k5CSrk6zeePv6MaYrSRrHWG/PVdXaYVVyAvDZabufDjx75HOiewL7AjcB706yHNjIVIgArgTOSnIP4PyqWjPGED5WVRuHVdNhwMeSbNq3+8hxn66qSrIO+ElVrQNI8k1gKfBg4NHAV4bH7wZcPvL484avVwHPmWlASfYC9q6qLw2bzgGesbljq+pMpuLL7kuW1VZnK0kay2w+Y7oAeCtTnxHdb2R7gOdW1XWjByd5A/ATplZbuwD/ClBVq5IcwdTbk+ckOb2qPgiM/uV+z2nPfdvwdRfgF8MqbnM2DF/vGrm96f4ipmL6xao6YSuP38jWvzeZNmZJ0jybzSX2ZwF/uWl1M+Ii4JQMS5skjx+27wX8uKruAl4A7Drsfyjw06r6B+D9wIHD8T9J8h+S7AIcu7kBVNUvge8l+U/DuZLkcbOYw1eB30/y8OHx907yiK085lfAnpsZyy+A9UmeOGw6cRbjkCTNgbEjVlU/qqp3bmbXm4B7AGuHS8zfNGw/AzgpyVeZeitx02rqSGBNkquB5wKbznka8BngYuDHMwzlROClSa4BvgkcPYs53Ay8CDg3yVqmovaorTzs08Cxm7uwA3gx8L+GCzvuGHcckqS5kSrfEZtPuy9ZVktOesekhzFR/qdYJM1WkquqasX07f7GDklSW0ZMktSWEZMktWXEJEltGTFJUluT+IW6d2v777MXq706T5LmhCsxSVJbRkyS1JYRkyS1ZcQkSW0ZMUlSW0ZMktSWEZMktWXEJEltGTFJUltGTJLUlhGTJLVlxCRJbRkxSVJbRkyS1JYRkyS1ZcQkSW0ZMUlSW0ZMktSWEZMktWXEJEltGTFJUltGTJLUlhGTJLVlxCRJbRkxSVJbRkyS1JYRkyS1ZcQkSW0ZMUlSW0ZMktSWEZMktWXEJEltGTFJUltGTJLUlhGTJLVlxCRJbRkxSVJbRkyS1JYRkyS1ZcQkSW0ZMUlSW0ZMktSWEZMktWXEJEltGTFJUltGTJLUlhGTJLVlxCRJbRkxSVJbRkyS1JYRkyS1ZcQkSW0ZMUlSW0ZMktSWEZMktWXEJEltGTFJUltGTJLUlhGTJLVlxCRJbRkxSVJbRkyS1JYRkyS1ZcQkSW0ZMUlSW0ZMktSWEZMktWXEJEltGTFJUltGTJLUlhGTJLVlxCRJbRkxSVJbRkyS1JYRkyS1ZcQkSW0ZMUlSW0ZMktSWEZMktWXEJEltGTFJUltGTJLUlhGTJLW1aNIDuLtZd+N6lp524aSHMS9uePNRkx6CpJ2cKzFJUltGTJLUlhGTJLVlxCRJbRkxSVJbRkyS1JYRkyS1taAilqSSnDNyf1GSm5N8ZhvPd0OSxZvZ/uwkp23HOG/dwvazkxy3reeVJM3OQvth59uAxya5V1XdATwNuHGun6SqLgAumOvzSpLm14JaiQ0+B2z6VQ8nAOdu2pHkkCSXJbl6+PrIYfuuSd6aZF2StUlOGTnfKUm+Pux71HD8i5K8e7h9dpJ3Dee7fnQlleQ1Sa4czvnG6QPNlHcnuTbJhcAD5vy7IUnaooUYsY8Axye5J3AAcMXIvm8DR1TV44HXA38zbD8ZeBjw+Ko6APjQyGNuqaoDgfcAp27hOZcATwSeBbwZIMnTgWXAIcBy4KAkR0x73LHAI4H9gZcBh23u5ElOTrI6yeqNt6+fefaSpLEttLcTqaq1SZYytQr77LTdewH/mGQZUMA9hu1PBd5bVXcO5/j5yGPOG75eBTxnC097flXdBVyb5IHDtqcPf64e7u/BVNRWjTzuCODcqtoI3JTk4i3M6UzgTIDdlyyrLYxBkjRLCy5igwuAtwJHAvcb2f4m4JKqOnYI3cphe5iK2uZsGL5uZMvz3TByOyNf/7aq/vdWxmqUJGlCFuLbiQBnAX9ZVeumbd+L31zo8aKR7V8AXp5kEUCSfz8HY7gIeEmSPYZz7pNk+mdeq5h663PXJEuAJ8/B80qSxrQgI1ZVP6qqd25m198Bf5vkK8CuI9vfB/wAWJvkGuD5czCGLwAfBi5Psg74OLDntMM+CfxfYB1Tn7l9aXufV5I0vlT5bth82n3Jslpy0jsmPYx54X9PTNJcSXJVVa2Yvn1BrsQkSRqHEZMktWXEJEltGTFJUltGTJLU1kL9Yeed1v777MVqr9qTpDnhSkyS1JYRkyS1ZcQkSW0ZMUlSW0ZMktSWEZMktWXEJEltGTFJUltGTJLUlhGTJLVlxCRJbRkxSVJbRkyS1JYRkyS1ZcQkSW0ZMUlSW0ZMktSWEZMktWXEJEltGTFJUltGTJLUlhGTJLVlxCRJbRkxSVJbRkyS1JYRkyS1ZcQkSW0ZMUlSW0ZMktSWEZMktWXEJEltGTFJUltGTJLUlhGTJLVlxCRJbRkxSVJbRkyS1JYRkyS1ZcQkSW0ZMUlSW0ZMktSWEZMktWXEJEltGTFJUltGTJLUlhGTJLVlxCRJbRkxSVJbRkyS1JYRkyS1ZcQkSW0ZMUlSW0ZMktSWEZMktWXEJEltGTFJUltGTJLUlhGTJLVlxCRJbRkxSVJbRkyS1JYRkyS1ZcQkSW0ZMUlSW0ZMktSWEZMktWXEJEltGTFJUltGTJLUlhGTJLVlxCRJbRkxSVJbRkyS1JYRkyS1ZcQkSW0ZMUlSW0ZMktSWEZMktWXEJEltGTFJUltGTJLUlhGTJLVlxCRJbRkxSVJbRkyS1JYRkyS1ZcQkSW0ZMUlSW0ZMktSWEZMktWXEJEltLZr0AGYjyUZgHVPj/hZwUlXdPtlRSZImpdtK7I6qWl5VjwX+DXj56M4ku87XQJK0+geAJO2MukVs1KXAw5McmeSSJB8G1iXZNcnpSa5MsjbJHwMkWZJkVZI1Sb6R5PDh2LOH++uSvGo4dmWSFcPtxUluGG6/KMnHknwa+EKS+yQ5a3iuq5McPZlvhSTdPbVcTQyroGcAnx82HQI8tqq+l+RkYH1VHZxkd+ArSb4APAe4qKr+elix3RtYDuwzrOxIsvcYT38ocEBV/TzJ3wAXV9VLhsd+Lcn/qarbpo33ZOBkgH333Xe75i5J+o1uK7F7JVkDrAZ+ALx/2P61qvrecPvpwAuH464A7gcsA64EXpzkDcD+VfUr4Hrgd5P8fZI/BH45xhi+WFU/H3mu04bnWgncE/itSlXVmVW1oqpW3P/+95/djCVJW9RtJXZHVS0f3ZAEYHTlE+CUqrpo+oOTHAEcBZyT5PSq+mCSxwH/EfgT4HnAS4A7+U3g7zntNNOf67lVdd02z0iStM26rcTGcRHwiiT3AEjyiOGzq4cCP62qf2BqBXdgksXALlX1CeAvgAOHc9wAHDTcPm4rz3VKhpImefycz0aStEXdVmLjeB+wFPj6EJebgWOAI4HXJPk1cCvwQmAf4ANJNsX8tcPXtwL/lOQFwMUzPNebgHcAa4fnugF41txNRZI0k1TVpMdwt7JixYpavXr1pIchSa0kuaqqVkzfvjO+nShJupswYpKktoyYJKktIyZJasuISZLaMmKSpLaMmCSpLSMmSWrLiEmS2jJikqS2jJgkqS0jJklqy4hJktoyYpKktoyYJKktIyZJasuISZLaMmKSpLaMmCSpLSMmSWrLiEmS2jJikqS2jJgkqS0jJklqy4hJktoyYpKktoyYJKmtVNWkx3C3kuRXwHWTHscOsBi4ZdKD2AF21nnBzjs359XLuPN6aFXdf/rGRXM/Hm3FdVW1YtKDmGtJVjuvXnbWuTmvXrZ3Xr6dKElqy4hJktoyYvPvzEkPYAdxXv3srHNzXr1s17y8sEOS1JYrMUlSW0ZsB0nyh0muS/LdJKdtZn+SvGvYvzbJgZMY52yNMa9HJbk8yYYkp05ijNtijHmdOLxOa5NcluRxkxjnbI0xr6OHOa1JsjrJEycxztna2rxGjjs4ycYkx83n+LbHGK/ZkUnWD6/ZmiSvn8Q4Z2uc12yY25ok30zypbFOXFX+meM/wK7APwO/C+wGXAM8etoxzwQ+BwT4PeCKSY97jub1AOBg4K+BUyc95jmc12HAfYfbz9iJXq89+M3HCgcA3570uOdiXiPHXQx8Fjhu0uOew9fsSOAzkx7rDpjX3sC1wL7D/QeMc25XYjvGIcB3q+r6qvo34CPA0dOOORr4YE35KrB3kiXzPdBZ2uq8quqnVXUl8OtJDHAbjTOvy6rq/w13vwo8eJ7HuC3GmdetNfyNAdwH6PAh+Tj//wI4BfgE8NP5HNx2Gndu3Ywzr+cD51XVD2Dq75JxTmzEdox9gB+O3P/RsG22xyw0Hcc8jtnO66VMraIXurHmleTYJN8GLgReMk9j2x5bnVeSfYBjgffO47jmwrj/Wzw0yTVJPpfkMfMztO0yzrweAdw3ycokVyV54Tgn9jd27BjZzLbp/8Id55iFpuOYxzH2vJI8mamIdfjsaKx5VdUngU8mOQJ4E/DUHT2w7TTOvN4B/HlVbUw2d/iCNc7cvs7Ur2C6NckzgfOBZTt6YNtpnHktAg4C/gC4F3B5kq9W1XdmOrER2zF+BDxk5P6DgZu24ZiFpuOYxzHWvJIcALwPeEZV/WyexrY9ZvV6VdWqJPslWVxVC/l39I0zrxXAR4aALQaemeTOqjp/Xka47bY6t6r65cjtzyY5Yyd5zX4E3FJVtwG3JVkFPA6YMWK+nbhjXAksS/KwJLsBxwMXTDvmAuCFw1WKvwesr6ofz/dAZ2mceXW01Xkl2Rc4D3jB1v5luICMM6+HZ/ibfrhCdjdgoQd6q/OqqodV1dKqWgp8HHhlg4DBeK/Zg0Zes0OY+nu8/WsGfAo4PMmiJPcGngB8a2sndiW2A1TVnUn+C3ARU1flnFVV30zy8mH/e5m6YuqZwHeB24EXT2q84xpnXkkeBKwG/h1wV5L/xtRVSL/c0nknbczX6/XA/YAzhr8/7qwF/stYx5zXc5n6x9SvgTuAPxq50GNBGnNeLY05t+OAVyS5k6nX7Pid4TWrqm8l+TywFrgLeF9VfWNr5/Y3dkiS2vLtRElSW0ZMktSWEZMktWXEJEltGTFJUltGTJLUlhGTJLVlxCRJbf1/y1fSaeshwyIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x648 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(6,9))\n",
    "\n",
    "ind = np.argsort(xgb_model.feature_importances_)[::-1]\n",
    "features_sorted = np.array(features)[ind]\n",
    "importances_sorted = xgb_model.feature_importances_[ind]\n",
    "\n",
    "plt.barh(y=range(len(features)), width=importances_sorted, height=0.2)\n",
    "plt.title('Gain')\n",
    "plt.yticks(ticks=range(len(features)), labels=features_sorted)\n",
    "plt.gca().invert_yaxis()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "a57de4cf-2c7d-4122-97cf-5c5e6e3a1b71",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d16fbcfd5880492e90ff4f0530e22ccc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1.0), Label(value='In progress...')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>#                              </th><th>MachineId  </th><th>MeasurementId  </th><th>PumpFailed  </th><th>SlowStart  </th><th>SlowEnd  </th><th>Pressure  </th><th>prediction_xgb  </th><th>prediction  </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td><i style='opacity: 0.6'>0</i>  </td><td>2113       </td><td>1312           </td><td>0.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td><td>1           </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>1</i>  </td><td>2120       </td><td>1319           </td><td>1.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>1               </td><td>1           </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>2</i>  </td><td>2121       </td><td>1319           </td><td>1.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>1               </td><td>1           </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3</i>  </td><td>2122       </td><td>1319           </td><td>1.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>1               </td><td>1           </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>4</i>  </td><td>2123       </td><td>-1             </td><td>0.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td><td>0           </td></tr>\n",
       "<tr><td>...                            </td><td>...        </td><td>...            </td><td>...         </td><td>...        </td><td>...      </td><td>...       </td><td>...             </td><td>...         </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>384</i></td><td>2203       </td><td>8232           </td><td>0.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td><td>1           </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>385</i></td><td>1500       </td><td>-1             </td><td>0.0         </td><td>0.0        </td><td>1.0      </td><td>0.0       </td><td>0               </td><td>0           </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>386</i></td><td>1501       </td><td>8336           </td><td>0.0         </td><td>0.0        </td><td>1.0      </td><td>0.0       </td><td>0               </td><td>1           </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>387</i></td><td>1502       </td><td>-1             </td><td>0.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td><td>0           </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>388</i></td><td>1503       </td><td>-1             </td><td>0.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td><td>0           </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "#    MachineId    MeasurementId    PumpFailed    SlowStart    SlowEnd    Pressure    prediction_xgb    prediction\n",
       "0    2113         1312             0.0           0.0          0.0        0.0         0                 1\n",
       "1    2120         1319             1.0           0.0          0.0        0.0         1                 1\n",
       "2    2121         1319             1.0           0.0          0.0        0.0         1                 1\n",
       "3    2122         1319             1.0           0.0          0.0        0.0         1                 1\n",
       "4    2123         -1               0.0           0.0          0.0        0.0         0                 0\n",
       "...  ...          ...              ...           ...          ...        ...         ...               ...\n",
       "384  2203         8232             0.0           0.0          0.0        0.0         0                 1\n",
       "385  1500         -1               0.0           0.0          1.0        0.0         0                 0\n",
       "386  1501         8336             0.0           0.0          1.0        0.0         0                 1\n",
       "387  1502         -1               0.0           0.0          0.0        0.0         0                 0\n",
       "388  1503         -1               0.0           0.0          0.0        0.0         0                 0"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from vaex.ml.sklearn import IncrementalPredictor\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "model = SGDClassifier(learning_rate='constant', eta0=0.001, random_state=42)\n",
    "vaex_model = IncrementalPredictor(features=features, target=target, model=model,\n",
    "                                  batch_size=11_000_000, partial_fit_kwargs={'classes':[0,1]})\n",
    "                            \n",
    "vaex_model.fit(df_train, progress='widget')\n",
    "\n",
    "df_train = vaex_model.transform(df_train)\n",
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "179a79ca-6984-4f9d-8b06-2933d8d29424",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metrics for the training set:\n",
      "Accuracy: 0.622\n",
      "f1 score: 0.358\n",
      "roc_auc:  0.560\n"
     ]
    }
   ],
   "source": [
    "print('Metrics for the training set:')\n",
    "binary_metrics(y_true=df_train.PumpFailed.values, y_pred=df_train.prediction.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "c1c93b4e-377f-4d5f-867c-b88eb9e134ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>#                             </th><th>MachineId  </th><th>MeasurementId  </th><th>PumpFailed  </th><th>SlowStart  </th><th>SlowEnd  </th><th>Pressure  </th><th>prediction_xgb  </th><th>prediction  </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td><i style='opacity: 0.6'>0</i> </td><td>33         </td><td>127            </td><td>0.0         </td><td>0.0        </td><td>1.0      </td><td>0.0       </td><td>0               </td><td>1           </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>1</i> </td><td>1000       </td><td>-1             </td><td>1.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td><td>0           </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>2</i> </td><td>1001       </td><td>-1             </td><td>1.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td><td>0           </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3</i> </td><td>1002       </td><td>-1             </td><td>1.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td><td>0           </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>4</i> </td><td>1003       </td><td>-1             </td><td>1.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td><td>0           </td></tr>\n",
       "<tr><td>...                           </td><td>...        </td><td>...            </td><td>...         </td><td>...        </td><td>...      </td><td>...       </td><td>...             </td><td>...         </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>95</i></td><td>2102       </td><td>-1             </td><td>1.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td><td>0           </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>96</i></td><td>2103       </td><td>1305           </td><td>1.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td><td>1           </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>97</i></td><td>2110       </td><td>-1             </td><td>0.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td><td>0           </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>98</i></td><td>2111       </td><td>-1             </td><td>0.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td><td>0           </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>99</i></td><td>2112       </td><td>-1             </td><td>0.0         </td><td>0.0        </td><td>0.0      </td><td>0.0       </td><td>0               </td><td>0           </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "#    MachineId    MeasurementId    PumpFailed    SlowStart    SlowEnd    Pressure    prediction_xgb    prediction\n",
       "0    33           127              0.0           0.0          1.0        0.0         0                 1\n",
       "1    1000         -1               1.0           0.0          0.0        0.0         0                 0\n",
       "2    1001         -1               1.0           0.0          0.0        0.0         0                 0\n",
       "3    1002         -1               1.0           0.0          0.0        0.0         0                 0\n",
       "4    1003         -1               1.0           0.0          0.0        0.0         0                 0\n",
       "...  ...          ...              ...           ...          ...        ...         ...               ...\n",
       "95   2102         -1               1.0           0.0          0.0        0.0         0                 0\n",
       "96   2103         1305             1.0           0.0          0.0        0.0         0                 1\n",
       "97   2110         -1               0.0           0.0          0.0        0.0         0                 0\n",
       "98   2111         -1               0.0           0.0          0.0        0.0         0                 0\n",
       "99   2112         -1               0.0           0.0          0.0        0.0         0                 0"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state = df_train.state_get()\n",
    "df_test.state_set(state)\n",
    "\n",
    "df_test.tail(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "138b1a74-52c3-4d12-b309-a641b86ea335",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.635\n",
      "f1 score: 0.384\n",
      "roc_auc:  0.571\n"
     ]
    }
   ],
   "source": [
    "binary_metrics(y_true=df_test.PumpFailed.values,y_pred=df_test.prediction.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b3e5335-28e8-4421-b136-87b3827dd2c3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be1f3374-ea37-4d61-bbb7-6a9f5e49dbae",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c922621a-1e19-46dd-84b0-883f0d676f07",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d659bf52-50f9-4058-adc7-d5c01ddde857",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee91eb70-5b62-46a9-b0fe-5c20d6d7cbcb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a42af15-53bc-45c2-aff0-700d785e788f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7174bcd3-5ca1-4893-9c0c-474bd0afdd66",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "environment": {
   "kernel": "python3",
   "name": "common-cpu.m91",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/base-cpu:m91"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}