Untitled12.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Untitled12.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyMW+PcP9yS+l6Uet4/VymbE",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/dimbasamster/Face-Detection-JavaScript-master/blob/master/Untitled12.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ApnuzI6zcAbu",
        "colab_type": "code",
        "outputId": "9af864bb-3f35-4753-f140-70a7701abc97",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "!pip install kaggle\n"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: kaggle in /usr/local/lib/python3.6/dist-packages (1.5.6)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.6/dist-packages (from kaggle) (2019.11.28)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from kaggle) (2.21.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from kaggle) (4.28.1)\n",
            "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.6/dist-packages (from kaggle) (2.6.1)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from kaggle) (1.24.3)\n",
            "Requirement already satisfied: six>=1.10 in /usr/local/lib/python3.6/dist-packages (from kaggle) (1.12.0)\n",
            "Requirement already satisfied: python-slugify in /usr/local/lib/python3.6/dist-packages (from kaggle) (4.0.0)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->kaggle) (2.8)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->kaggle) (3.0.4)\n",
            "Requirement already satisfied: text-unidecode>=1.3 in /usr/local/lib/python3.6/dist-packages (from python-slugify->kaggle) (1.3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B3_HsBwWcQUu",
        "colab_type": "code",
        "outputId": "2511d999-8b9e-4c4e-f093-6be32ed3d9b3",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 91
        }
      },
      "source": [
        "from google.colab import files\n",
        "files.upload()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-043e64a1-aee1-467f-9a04-148dca5c89c2\" name=\"files[]\" multiple disabled />\n",
              "     <output id=\"result-043e64a1-aee1-467f-9a04-148dca5c89c2\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving kaggle.json to kaggle.json\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'kaggle.json': b'{\"username\":\"misakrug\",\"key\":\"57326d828973dfc30e8b106b392b22a5\"}'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yFgmOIhOhKbJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!mkdir -p ~/.kaggle\n",
        "!cp kaggle.json ~/.kaggle/\n",
        "!chmod 600 ~/.kaggle/kaggle.json"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ra1-UptdiHmR",
        "colab_type": "code",
        "outputId": "ef1b3cba-5e1a-4e9a-86f6-6312d632efac",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "!cp"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cp: missing file operand\n",
            "Try 'cp --help' for more information.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xuZraA1Tg5R0",
        "colab_type": "code",
        "outputId": "55f777ec-46bb-45d6-a331-9b3bebb62f71",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 646
        }
      },
      "source": [
        "!kaggle competitions download -c bengaliai-cv19"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Warning: Looks like you're using an outdated API Version, please consider updating (server 1.5.6 / client 1.5.4)\n",
            "Downloading train_image_data_0.parquet.zip to /content\n",
            " 99% 983M/991M [00:07<00:00, 171MB/s]\n",
            "100% 991M/991M [00:07<00:00, 144MB/s]\n",
            "Downloading train_image_data_3.parquet.zip to /content\n",
            " 99% 983M/993M [00:09<00:00, 73.5MB/s]\n",
            "100% 993M/993M [00:09<00:00, 107MB/s] \n",
            "Downloading test_image_data_1.parquet.zip to /content\n",
            "  0% 0.00/1.30M [00:00<?, ?B/s]\n",
            "100% 1.30M/1.30M [00:00<00:00, 181MB/s]\n",
            "Downloading test.csv to /content\n",
            "  0% 0.00/1.70k [00:00<?, ?B/s]\n",
            "100% 1.70k/1.70k [00:00<00:00, 1.51MB/s]\n",
            "Downloading sample_submission.csv to /content\n",
            "  0% 0.00/944 [00:00<?, ?B/s]\n",
            "100% 944/944 [00:00<00:00, 916kB/s]\n",
            "Downloading test_image_data_0.parquet.zip to /content\n",
            "  0% 0.00/1.18M [00:00<?, ?B/s]\n",
            "100% 1.18M/1.18M [00:00<00:00, 167MB/s]\n",
            "Downloading class_map.csv to /content\n",
            "  0% 0.00/4.72k [00:00<?, ?B/s]\n",
            "100% 4.72k/4.72k [00:00<00:00, 4.48MB/s]\n",
            "Downloading train_image_data_2.parquet.zip to /content\n",
            "100% 990M/992M [00:11<00:00, 56.4MB/s]\n",
            "100% 992M/992M [00:11<00:00, 91.1MB/s]\n",
            "Downloading test_image_data_3.parquet.zip to /content\n",
            "  0% 0.00/1.28M [00:00<?, ?B/s]\n",
            "100% 1.28M/1.28M [00:00<00:00, 154MB/s]\n",
            "Downloading test_image_data_2.parquet.zip to /content\n",
            "  0% 0.00/1.25M [00:00<?, ?B/s]\n",
            "100% 1.25M/1.25M [00:00<00:00, 176MB/s]\n",
            "Downloading train.csv.zip to /content\n",
            "  0% 0.00/1.32M [00:00<?, ?B/s]\n",
            "100% 1.32M/1.32M [00:00<00:00, 89.3MB/s]\n",
            "Downloading train_image_data_1.parquet.zip to /content\n",
            "100% 981M/986M [00:16<00:00, 40.0MB/s]\n",
            "100% 986M/986M [00:16<00:00, 63.2MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9pk1uJd2hT7d",
        "colab_type": "code",
        "outputId": "851ff803-43e3-4a75-946c-f4ea05ecdf23",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "!unzip bengaliai-cv19 "
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "unzip:  cannot find or open bengaliai-cv19, bengaliai-cv19.zip or bengaliai-cv19.ZIP.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-bY2_huX3VBw",
        "colab_type": "code",
        "outputId": "01a443b2-8abf-4903-9744-babb5b264958",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "pip install iterative-stratification"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting iterative-stratification\n",
            "  Downloading https://files.pythonhosted.org/packages/9d/79/9ba64c8c07b07b8b45d80725b2ebd7b7884701c1da34f70d4749f7b45f9a/iterative_stratification-0.1.6-py3-none-any.whl\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from iterative-stratification) (1.4.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from iterative-stratification) (1.17.5)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from iterative-stratification) (0.22.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->iterative-stratification) (0.14.1)\n",
            "Installing collected packages: iterative-stratification\n",
            "Successfully installed iterative-stratification-0.1.6\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a2r7U0glvFBe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "from iterstrat.ml_stratifiers import MultilabelStratifiedKFold\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    df = pd.read_csv(\"/content/train.csv.zip\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pRJ8TDEz5AAV",
        "colab_type": "code",
        "outputId": "5b40b8ca-c5bf-44a6-c68f-be6f8d6b8683",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        }
      },
      "source": [
        "print(df.head())"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  image_id  grapheme_root  vowel_diacritic  consonant_diacritic grapheme\n",
            "0  Train_0             15                9                    5   ক্ট্রো\n",
            "1  Train_1            159                0                    0        হ\n",
            "2  Train_2             22                3                    5     খ্রী\n",
            "3  Train_3             53                2                    2     র্টি\n",
            "4  Train_4             71                9                    5     থ্রো\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M1kVMPPc5n4D",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df.loc[:, 'kfold'] =-1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7QD8PtSH6Akc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df=df.sample(frac=1).reset_index(drop=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SWuvzkJM8WsR",
        "colab_type": "code",
        "outputId": "0e3bcd13-be54-4226-d9df-cefd1d3f85a4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "X = df.image_id.values\n",
        "y = df[[\"grapheme_root\", \"vowel_diacritic\", \"consonant_diacritic\"]].values\n",
        "mskf = MultilabelStratifiedKFold(n_splits=5)\n",
        "\n",
        "for fold, (trn_, val_) in enumerate(mskf.split(X, y)):\n",
        "    print(\"TRAIN: \", trn_, \"VAL: \", val_)\n",
        "    df.loc[val_, \"kfold\"] = fold\n",
        "\n",
        "\n",
        "print(df.kfold.value_counts())\n",
        "df.to_csv(\"/content/train_folds.csv\", index=False)\n",
        "\n"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TRAIN:  [     0      1      2 ... 200837 200838 200839] VAL:  [    11     18     21 ... 200829 200830 200832]\n",
            "TRAIN:  [     1      5      7 ... 200837 200838 200839] VAL:  [     0      2      3 ... 200825 200834 200835]\n",
            "TRAIN:  [     0      1      2 ... 200836 200837 200839] VAL:  [     5     13     33 ... 200823 200833 200838]\n",
            "TRAIN:  [     0      1      2 ... 200836 200837 200838] VAL:  [     7     14     31 ... 200824 200831 200839]\n",
            "TRAIN:  [     0      2      3 ... 200835 200838 200839] VAL:  [     1     10     15 ... 200827 200836 200837]\n",
            "4    40168\n",
            "3    40168\n",
            "2    40168\n",
            "1    40168\n",
            "0    40168\n",
            "Name: kfold, dtype: int64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ou5LnfZdgr3I",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "4b8ee933-a036-43bb-a644-46ac944b2a13"
      },
      "source": [
        "!unzip train.csv.zip"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  train.csv.zip\n",
            "  inflating: train.csv               \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BWZf1j63k2B1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "8a5ac6ea-51f8-485f-8646-dc9a64cddf05"
      },
      "source": [
        "!unzip /content/test_image_data_0.parquet.zip"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/test_image_data_0.parquet.zip\n",
            "  inflating: test_image_data_0.parquet  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iWqi1kFXlPL6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        },
        "outputId": "91d21f9e-8b58-4e08-d5f7-086e7990ffcb"
      },
      "source": [
        "!unzip /content/test_image_data_1.parquet.zip.parquet.zip\n",
        "!unzip /content/test_image_data_2.parquet.zip.parquet.zip\n",
        "!unzip /content/test_image_data_3.parquet.zip.parquet.zip\n",
        "\n"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "unzip:  cannot find or open /content/test_image_data_1.parquet.zip.parquet.zip, /content/test_image_data_1.parquet.zip.parquet.zip.zip or /content/test_image_data_1.parquet.zip.parquet.zip.ZIP.\n",
            "unzip:  cannot find or open /content/test_image_data_2.parquet.zip.parquet.zip, /content/test_image_data_2.parquet.zip.parquet.zip.zip or /content/test_image_data_2.parquet.zip.parquet.zip.ZIP.\n",
            "unzip:  cannot find or open /content/test_image_data_3.parquet.zip.parquet.zip, /content/test_image_data_3.parquet.zip.parquet.zip.zip or /content/test_image_data_3.parquet.zip.parquet.zip.ZIP.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eBJm7FGql9VJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "24a5bb83-453f-403c-e446-6a3ed7ccc30f"
      },
      "source": [
        "!unzip /content/train_image_data_1.parquet.zip"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/train_image_data_1.parquet.zip\n",
            "  inflating: train_image_data_1.parquet  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AHiQsxjjmCoZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "883febc6-1565-4707-8823-dca9bda35a54"
      },
      "source": [
        "!unzip /content/train_image_data_0.parquet.zip"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/train_image_data_0.parquet.zip\n",
            "  inflating: train_image_data_0.parquet  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-q0MqnR4mdEQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "805adb42-7dd4-4577-e122-e775e2a8046e"
      },
      "source": [
        "!unzip /content/train_image_data_1.parquet.zip"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/train_image_data_1.parquet.zip\n",
            "replace train_image_data_1.parquet? [y]es, [n]o, [A]ll, [N]one, [r]ename: n\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cjEPLH1hmt-R",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "56603915-caa3-4c19-d583-2873e81318cd"
      },
      "source": [
        "!unzip /content/train_image_data_2.parquet.zip"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/train_image_data_2.parquet.zip\n",
            "  inflating: train_image_data_2.parquet  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1rzKDLlCnAwJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "dffd11f3-6a96-4aaa-cbbb-b419e4b9c49d"
      },
      "source": [
        "!unzip /content/train_image_data_3.parquet.zip"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/train_image_data_3.parquet.zip\n",
            "  inflating: train_image_data_3.parquet  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6XnqKkAUnZwK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "ef109597-95ff-404c-f1e7-6edf481c91c9"
      },
      "source": [
        "!unzip /content/train_image_data_1.parquet.zip"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/train_image_data_1.parquet.zip\n",
            "replace train_image_data_1.parquet? [y]es, [n]o, [A]ll, [N]one, [r]ename: n\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rQ4aS6feny2Q",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "3182f897-da9f-4d86-bee3-a92f1b025bc0"
      },
      "source": [
        "!unzip /content/train_image_data_2.parquet.zip"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/train_image_data_2.parquet.zip\n",
            "replace train_image_data_2.parquet? [y]es, [n]o, [A]ll, [N]one, [r]ename: n\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "itrvLvgooUFz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "2b348075-0545-4adb-8b8f-53c341a62559"
      },
      "source": [
        "!unzip /content/train_image_data_3.parquet.zip"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/train_image_data_3.parquet.zip\n",
            "replace train_image_data_3.parquet? [y]es, [n]o, [A]ll, [N]one, [r]ename: n\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H3A_Jn6VNmwZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7PKYQ3CTO5mb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3gxRL_PvPCbf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import joblib"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_5b7y8AgPH4W",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import glob"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TyWebSsSPMdd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tqdm import tqdm"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SH0Q9Ky8PUiq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 333
        },
        "outputId": "6b0567de-a283-47c7-eb43-667831e9338b"
      },
      "source": [
        "if __name__ == \"__main__\":\n",
        "    files = glob.glob(\"/content/train_*.parquet\")\n",
        "    for f in files:\n",
        "        df = pd.read_parquet(f)\n",
        "        image_ids = df.image_id.values\n",
        "        df = df.drop(\"image_id\", axis=1)\n",
        "        image_array = df.values\n",
        "        for j, img_id in tqdm(enumerate(image_ids), total=len(image_ids)):\n",
        "            joblib.dump(image_array[j, :], f\"../content/image_pickles/{img_id}.pkl\")\n"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r  0%|          | 0/50210 [00:00<?, ?it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "error",
          "ename": "FileNotFoundError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-39-46e90aa1542c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m         \u001b[0mimage_array\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mimg_id\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimage_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimage_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m             \u001b[0mjoblib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimage_array\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mf\"../content/image_pickles/{img_id}.pkl\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36mdump\u001b[0;34m(value, filename, compress, protocol, cache_size)\u001b[0m\n\u001b[1;32m    502\u001b[0m             \u001b[0mNumpyPickler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprotocol\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprotocol\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    503\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mis_filename\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 504\u001b[0;31m         \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'wb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    505\u001b[0m             \u001b[0mNumpyPickler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprotocol\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprotocol\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    506\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '../content/image_pickles/Train_50210.pkl'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RC9554DMYRg6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 129
        },
        "outputId": "90317855-843e-4a82-db86-8de3e92411a4"
      },
      "source": [
        "python create_image_pickles.py"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "error",
          "ename": "SyntaxError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-40-c08dbad7f9c9>\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m    python create_image_pickles.py\u001b[0m\n\u001b[0m                              ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7UtKwsN5WHV5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PwI9CW5mNvKs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = pd.read_parquet(\"/content/train_image_data_0.parquet\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d3fvkf-zFoFX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import joblib\n",
        "from PIL import Image\n",
        "import albumentations\n",
        "import numpy as np\n",
        "\n",
        "\n",
        "class BengaliDatasetTrain:\n",
        "    def __init__(self, folds, img_height, img_width, mean, std):\n",
        "        df = pd.read_csv(\"../content/train_folds/csv\")\n",
        "        df = df[[\"image_id\", \"graphem_root\", \"vowel_diacritic\", \"consonant_diacritic\", \"kfold\"]]\n",
        "        df = df[df.kfold.isin(folds)].reset_index(drop=true)\n",
        "        self.image_ids = df.image_id.values\n",
        "        self.graphem_root = df.graphem_root.values\n",
        "        self.vowel_diacritic = df.vowel_diacritic.values\n",
        "        self.consonant_diacritic = df.consonant_diacritic.values\n",
        "\n",
        "        if len(folds) == 1:\n",
        "           self.aug = albumentations.compose([\n",
        "                   albumentations.Resize(img_height, img_width, always_apply=True),\n",
        "                    albumentations.Normalize(mean, std, always_apply=True)                        \n",
        "           ])\n",
        "           else:\n",
        "               self.aug = albumentations.compose([\n",
        "                    albumentations.Resize(img_height, img_width, always_apply=True),\n",
        "                    albumentations.ShiftScaleRotate(shift_limit=0.0625,\n",
        "                                                    scale_limit=0.1,\n",
        "                                                    rotate_limit=5,\n",
        "                                                    p=0.9),\n",
        "                    albumentations.Normalize(mean, std, always_apply=True)                    \n",
        "        ])\n",
        "\n",
        "        \n",
        "def __len__(self):\n",
        "    return len(self.image_ids)\n",
        "\n",
        "def __getitem__(self,item):\n",
        "    image = joblib.load(f\"../content/image_pickles/{self.image_ids[item]}.pkl\")\n",
        "    image = image.reshape(137, 236).astype(float)\n",
        "    image = Image.fromarray(image).convert(\"RGB\")\n",
        "    image = self.aug(image=nparray(image))[\"image\"]\n",
        "    image = np.transpose(image, (2, 0, 1)).astype(np.float32)\n",
        "    return {\n",
        "        'image': torch.tensor(image, dtype=torch.float),\n",
        "        'graphem_root': torch.tensor(self.graphem__root[item], dtype=torch.long),\n",
        "         'vowel_diacritic': torch.tensor(self.vowel_diacritic[item], dtype=torch.long),\n",
        "          'consonant_diacritic': torch.tensor(self.consonant_diacritic[item], dtype=torch.long),\n",
        "    }\n",
        "\n",
        "    \n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}