gdrive_downloader.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import io\n",
    "import os\n",
    "import oauth2client\n",
    "from oauth2client import file, client, tools\n",
    "from googleapiclient.discovery import build\n",
    "from googleapiclient.http import MediaIoBaseDownload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import ipywidgets as widgets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Class to wrap Google Drive API "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DriveDownloader():\n",
    "    \"\"\"Thin wrapper to easily automate Google Drive authenticating, searching and downloading\n",
    "    \n",
    "    Google Drive API v3 tutorial: https://developers.google.com/drive/api/v3/quickstart/python \n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self):\n",
    "        \"\"\"Setting attribute defaults (for using the public demo gdrive folder)\"\"\"\n",
    "\n",
    "        self.folder_name = 'test1' \n",
    "        self.folder_id = '17VB4dXCU-FhW_qNoUphmrjjGJ2brx3f1'  # https://drive.google.com/drive/u/0/folders/17VB4dXCU-FhW_qNoUphmrjjGJ2brx3f1\n",
    "        self.search_key = '.md'\n",
    "        self.path = './data/'\n",
    "        self.service = None\n",
    "        self.items = None\n",
    "\n",
    "    \n",
    "    def authenticate_gdrive(self, path_to_credentials):\n",
    "        \"\"\"Abstracting the authentication process\n",
    "        Reference:\n",
    "        https://medium.com/@umdfirecoml/a-step-by-step-guide-on-how-to-download-your-google-drive-data-to-your-jupyter-notebook-using-the-52f4ce63c66c\n",
    "        \"\"\"\n",
    "        \n",
    "        if self.service is None:\n",
    "            os.makedirs(path_to_credentials, exist_ok=True)\n",
    "            obj = lambda: None\n",
    "            lmao = {\"auth_host_name\":'localhost', 'noauth_local_webserver':'store_true', 'auth_host_port':[8080, 8090], 'logging_level':'ERROR'}\n",
    "            for k, v in lmao.items():\n",
    "                setattr(obj, k, v)\n",
    "\n",
    "            # authorization boilerplate code\n",
    "            SCOPES = 'https://www.googleapis.com/auth/drive.readonly'\n",
    "            store = file.Storage(path_to_credentials + 'token.json')\n",
    "            creds = store.get()\n",
    "            # The following will give you a link if token.json does not exist, the link allows the user to give this app permission\n",
    "            if not creds or creds.invalid:\n",
    "                flow = client.flow_from_clientsecrets(path_to_credentials + 'client_id.json', SCOPES)\n",
    "                creds = tools.run_flow(flow, store, obj)\n",
    "\n",
    "            service = build('drive', 'v3', credentials=creds)\n",
    "            self.service = service\n",
    "\n",
    "    @staticmethod\n",
    "    def search_gdrive(service, folder_id, search_key):\n",
    "        \"\"\"Search based on a parent folder_id and a regex search_key\n",
    "        \n",
    "        Will print out the search query and the results\n",
    "        Returns: the list of items found mathing the criteria\n",
    "        \"\"\"\n",
    "        \n",
    "        if service is None:\n",
    "            print(\"[error in search_gdrive]: no service. run authenticate_gdrive() first\")\n",
    "            return None\n",
    "        else:\n",
    "            query_string = f\"name contains '{search_key}' and parents in '{folder_id}'\"\n",
    "            print(query_string)\n",
    "            results = service.files().list(\n",
    "                    q=query_string,\n",
    "                    spaces='drive',\n",
    "                    pageSize=50, fields=\"nextPageToken, files(id, name)\").execute()\n",
    "            items = results.get('files', [])\n",
    "\n",
    "            if not items:\n",
    "                print('No files found.')\n",
    "            else:\n",
    "                print('Files:')\n",
    "                for i, item in enumerate(items):\n",
    "                    print(f\"item_id: {i}, filename: {item['name']} , file_id: {item['id']}\")\n",
    "\n",
    "            return items\n",
    "\n",
    "    @staticmethod\n",
    "    def download_gdrive(service, items, path, folder_name):\n",
    "        \"\"\"Download the files in an items list\n",
    "        \n",
    "        Will print the percentage compeltion during download of each file\n",
    "        \"\"\"\n",
    "        \n",
    "        path_to_save = path + folder_name + '/'\n",
    "        os.makedirs(path_to_save, exist_ok=True)\n",
    "        \n",
    "        if items is None or items==[]:\n",
    "            print(\"[error in download_gdrive]: no items to download. Run search_gdrive() first\")\n",
    "        else:\n",
    "            for i, item in enumerate(items): \n",
    "                file_id = item['id']\n",
    "                filename = item['name']\n",
    "\n",
    "                request = service.files().get_media(fileId=file_id)\n",
    "                fh = io.FileIO(path_to_save + filename, mode='w')\n",
    "\n",
    "                downloader = MediaIoBaseDownload(fh, request)\n",
    "                done = False\n",
    "                while done is False:\n",
    "                    status, done = downloader.next_chunk()\n",
    "                    print(f\"[{i+1}/{len(items)}] Download {int(status.progress() * 100)}%\")\n",
    "            print(f\"Download of {len(items)} files complete\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Class to combine Google Drive API with ipywidgets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class WidgetDriveDownloader(DriveDownloader):\n",
    "    \"\"\"Class that adds the convenient ipywidget functionality\n",
    "   \n",
    "   Populates the defaults for searching and downloading.\n",
    "   Makes the repetitive (but not qutie scriptable) process of \n",
    "   downloading particular files painless.\n",
    "   \"\"\"\n",
    "    \n",
    "    def __init__(self):\n",
    "        \"\"\"Inherit and setup the UI\"\"\"\n",
    "\n",
    "        super().__init__()\n",
    "        self.setup_widgets()\n",
    "        \n",
    "\n",
    "    def setup_widgets(self):\n",
    "        \"\"\"ipywidget setup: textboxes and buttons\"\"\"\n",
    "        \n",
    "        # first: make input textboxes for user defined parameters\n",
    "        self.ui = widgets.VBox([\n",
    "            widgets.Text(\n",
    "                    description='Folder name',\n",
    "                    placeholder=f'eg: {self.folder_name}', \n",
    "                    value=f\"{self.folder_name}\" # use the __init__ values as defaults\n",
    "                    \n",
    "            ),\n",
    "            widgets.Text(\n",
    "                description='Folder ID',\n",
    "                placeholder=f'eg: {self.folder_id}', \n",
    "                value=f\"{self.folder_id}\" # use the __init__ values as defaults\n",
    "            ),\n",
    "            widgets.Text(\n",
    "                description='Search Key',\n",
    "                placeholder=f'eg: {self.search_key}',     \n",
    "                value=f\"{self.search_key}\" # use the __init__ values as defaults\n",
    "                \n",
    "            ), \n",
    "             widgets.Text(\n",
    "                description='Path to save',\n",
    "                placeholder=f'eg: {self.path}', \n",
    "                value=f\"{self.path}\" # use the __init__ values as defaults\n",
    "            ), \n",
    "        ])   \n",
    "        self.out = widgets.interactive_output(\n",
    "            self.set_inputs, {\n",
    "                'folder_name': self.ui.children[0],\n",
    "                'folder_id': self.ui.children[1], \n",
    "                'search_key': self.ui.children[2], \n",
    "                'path': self.ui.children[3],\n",
    "            }\n",
    "        )\n",
    "\n",
    "        # next, make buttons for starting the search and starting the download\n",
    "        self.button_search = widgets.Button(\n",
    "            button_style='success', \n",
    "            tooltip='Click to search gdrive folder ID for files matching Search Key',\n",
    "            description='Search gdrive',\n",
    "        )\n",
    "        self.button_search.on_click(self.search)\n",
    "\n",
    "        self.button_download = widgets.Button(\n",
    "            button_style='success', \n",
    "            tooltip='Click to download the files to the Path to save',\n",
    "            description='Download files',\n",
    "        )\n",
    "        self.button_download.on_click(self.download)\n",
    "        self.output = widgets.Output()\n",
    "        \n",
    "\n",
    "    def set_inputs(self, folder_name, folder_id, search_key, path):\n",
    "        \"\"\"callback for mapping the input textboxes to the object attributes\"\"\"\n",
    "        \n",
    "        self.folder_name = folder_name\n",
    "        self.folder_id = folder_id\n",
    "        self.search_key = search_key\n",
    "        self.path = path\n",
    "        print((self.folder_name, self.folder_id, self.search_key, self.path))\n",
    "\n",
    "\n",
    "    def search(self, b):\n",
    "        \"\"\"button callback for wrapping the gdrive search method\"\"\"\n",
    "        \n",
    "        with self.output:\n",
    "            self.output.clear_output()\n",
    "            print(\"searching...\")\n",
    "            self.items = self.search_gdrive(self.service, self.folder_id, self.search_key)\n",
    "\n",
    "    def download(self, b):\n",
    "        \"\"\"button callback for wrapping the gdrive download method\"\"\"\n",
    "\n",
    "        with self.output:\n",
    "            self.output.clear_output()\n",
    "            print(\"downloading...\")\n",
    "            self.download_gdrive(self.service, self.items, self.path, self.folder_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dashboard to search for files in a gdrive Folder ID, download those that match a Search Key and save them to a Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "downloader = WidgetDriveDownloader()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# follow the instruction to get the client_id.json credentials\n",
    "# reference: https://medium.com/@umdfirecoml/a-step-by-step-guide-on-how-to-download-your-google-drive-data-to-your-jupyter-notebook-using-the-52f4ce63c66c\n",
    "# and choose the folder where you store the credentials as \"path_to_credentials\"\n",
    "downloader.authenticate_gdrive(path_to_credentials='/home/ubuntu/utilities/GoogleDrive_downloader_ipywidgets/credentials/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Type your search query into the input boxes below:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a2cf345442c5436aaa788227b667c05f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(Text(value='test1', description='Folder name', placeholder='eg: test1'), Text(value='17VB4dXCU-…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "165cb5b1064347ec84fb793c2f349f67",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Then click the buttons to search and then download:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dea45b0f8479487fb12edc59979be0b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Button(button_style='success', description='Search gdrive', style=ButtonStyle(), tooltip='Click to search gdri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "120a45addf1446efa532b8ff0a64f5b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Button(button_style='success', description='Download files', style=ButtonStyle(), tooltip='Click to download t…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "84654a7839c645ab91507338e81ee7c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"Type your search query into the input boxes below:\")\n",
    "display(downloader.ui, downloader.out)\n",
    "print(\"Then click the buttons to search and then download:\")\n",
    "display(downloader.button_search, downloader.button_download, downloader.output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ls: cannot access './data/test1': No such file or directory\r\n"
     ]
    }
   ],
   "source": [
    "# check the file names and file sizes that were downloaded\n",
    "!ls -lath {downloader.path + downloader.folder_name}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find . -name \"*.zip\" -exec unar -d {} \\;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### Useful folder ID / folder name combinations for our project\n",
    "\n",
    "# folder_id = '1k_VWJF7XEdJlrxgRqOh5nZKusbpTdUJe'  # TsushimaKatuki_sampleTasking\n",
    "# folder_id = '192J6teGXae-BnM57r9GNUadeBf-amnAS'  # shimane_sampleTasking\n",
    "# folder_id = '1TLLxCSAepRXM3PDtUoVMDNtdBzHBjkxz'  # yonaguni_SampleTasking\n",
    "# folder_id = '1sw9xgzmVULTJVcVHrf_xpq7i5o9Qy3EN'  # Shimane-28052020-1_PAN\n",
    "# folder_id = '1cTpBmP3vrZ8sfxzZSsIo1TUamTPUzuFg'  # Shimane-28052020-1_MS\n",
    "# folder_id = '1GSBv7LlQCvv0Nt9HqgF6Pd77oeCRUM2z'  # Shimane-28052020-2_PAN\n",
    "# folder_id = '1T-CHk7eUarJVqJDSg06OIvKtRIPiv25W'  # Shimane-28052020-2_MS\n",
    "\n",
    "\n",
    "# search_key = '.tif'\n",
    "# search_key = '.xlsx'"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}