data_retrieval.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Retrieves list of songs from SpotifyCharts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fycharts.SpotifyCharts import SpotifyCharts\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO : 10/03/2020 02:50:01 PM : Extracting top 200 daily for 2020-03-10 - us\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-21-9d5448a4289b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mapi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSpotifyCharts\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m#api.top200Daily(output_file = \"top_200_daily.csv\", region = [\"us\"])\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mapi\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtop200Daily\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_file\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"top_200_2020-03-10.csv\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstart\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"2020-03-10\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"2020-03-10\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mregion\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m\"us\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/fycharts/SpotifyCharts.py\u001b[0m in \u001b[0;36mtop200Daily\u001b[0;34m(self, output_file, output_db, webhook, start, end, region)\u001b[0m\n\u001b[1;32m    183\u001b[0m                         \u001b[0mk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    184\u001b[0m                         \u001b[0;32mfor\u001b[0m \u001b[0mregion\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mregions\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 185\u001b[0;31m                                 \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhelperTop200Daily\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtheRange\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mregion\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    186\u001b[0m                                 \u001b[0;32mif\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_file\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    187\u001b[0m                                         \u001b[0mdict_for_csv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m\"df\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"out_file\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mfile\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"j\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/fycharts/crawler_base.py\u001b[0m in \u001b[0;36mhelperTop200Daily\u001b[0;34m(self, date, region)\u001b[0m\n\u001b[1;32m    125\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Extracting top 200 daily for {} - {}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mregion\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    126\u001b[0m                 \u001b[0murl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"https://spotifycharts.com/regional/{}/daily/{}/download\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mregion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 127\u001b[0;31m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__makeRequests\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mregion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m200\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    128\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    129\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/fycharts/crawler_base.py\u001b[0m in \u001b[0;36m__makeRequests\u001b[0;34m(self, url, date, region, isSkip, size)\u001b[0m\n\u001b[1;32m     72\u001b[0m                                 \u001b[0ms\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrequests\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     73\u001b[0m                                 \u001b[0ms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmount\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"https://\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mHTTPAdapter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmax_retries\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mretries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 74\u001b[0;31m                                 \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     75\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     76\u001b[0m                                 \u001b[0;32mif\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mres\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus_code\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m200\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/requests/sessions.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, url, **kwargs)\u001b[0m\n\u001b[1;32m    544\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    545\u001b[0m         \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msetdefault\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'allow_redirects'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 546\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'GET'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    547\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    548\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/requests/sessions.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m    531\u001b[0m         }\n\u001b[1;32m    532\u001b[0m         \u001b[0msend_kwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msettings\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 533\u001b[0;31m         \u001b[0mresp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0msend_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    534\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    535\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/requests/sessions.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    644\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    645\u001b[0m         \u001b[0;31m# Send the request\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 646\u001b[0;31m         \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0madapter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    647\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    648\u001b[0m         \u001b[0;31m# Total elapsed time of the request (approximately)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/requests/adapters.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    447\u001b[0m                     \u001b[0mdecode_content\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    448\u001b[0m                     \u001b[0mretries\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_retries\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 449\u001b[0;31m                     \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    450\u001b[0m                 )\n\u001b[1;32m    451\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36murlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    729\u001b[0m                 \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpool_timeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpool_timeout\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    730\u001b[0m                 \u001b[0mrelease_conn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrelease_conn\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 731\u001b[0;31m                 body_pos=body_pos, **response_kw)\n\u001b[0m\u001b[1;32m    732\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    733\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36murlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    729\u001b[0m                 \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpool_timeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpool_timeout\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    730\u001b[0m                 \u001b[0mrelease_conn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrelease_conn\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 731\u001b[0;31m                 body_pos=body_pos, **response_kw)\n\u001b[0m\u001b[1;32m    732\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    733\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36murlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    729\u001b[0m                 \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpool_timeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpool_timeout\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    730\u001b[0m                 \u001b[0mrelease_conn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrelease_conn\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 731\u001b[0;31m                 body_pos=body_pos, **response_kw)\n\u001b[0m\u001b[1;32m    732\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    733\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36murlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    729\u001b[0m                 \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpool_timeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpool_timeout\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    730\u001b[0m                 \u001b[0mrelease_conn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrelease_conn\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 731\u001b[0;31m                 body_pos=body_pos, **response_kw)\n\u001b[0m\u001b[1;32m    732\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    733\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/urllib3/connectionpool.py\u001b[0m in \u001b[0;36murlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    721\u001b[0m             \u001b[0mdrain_and_release_conn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    722\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 723\u001b[0;31m             \u001b[0mretries\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    724\u001b[0m             \u001b[0mlog\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdebug\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Retry: %s\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    725\u001b[0m             return self.urlopen(\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/urllib3/util/retry.py\u001b[0m in \u001b[0;36msleep\u001b[0;34m(self, response)\u001b[0m\n\u001b[1;32m    280\u001b[0m                 \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    281\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 282\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sleep_backoff\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    283\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    284\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_is_connection_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/urllib3/util/retry.py\u001b[0m in \u001b[0;36m_sleep_backoff\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    264\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mbackoff\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    265\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 266\u001b[0;31m         \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbackoff\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    267\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    268\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "api = SpotifyCharts()\n",
    "#api.top200Daily(output_file = \"top_200_daily.csv\", region = [\"us\"])\n",
    "api.top200Daily(output_file = \"top_200_2020-03-10.csv\", start = \"2020-03-10\", end = \"2020-03-10\", region = [\"us\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"top_200_daily.csv\")\n",
    "df = df.iloc[0:200]\n",
    "df[\"Position\"] = pd.to_numeric(df[\"Position\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Position</th>\n",
       "      <th>Track Name</th>\n",
       "      <th>Artist</th>\n",
       "      <th>Streams</th>\n",
       "      <th>date</th>\n",
       "      <th>region</th>\n",
       "      <th>spotify_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Bad and Boujee (feat. Lil Uzi Vert)</td>\n",
       "      <td>Migos</td>\n",
       "      <td>1371493</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>us</td>\n",
       "      <td>4Km5HrUvYTaSUfiSGPJeQR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>Fake Love</td>\n",
       "      <td>Drake</td>\n",
       "      <td>1180074</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>us</td>\n",
       "      <td>343YBumqHu19cGoGARUTsd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>Starboy</td>\n",
       "      <td>The Weeknd</td>\n",
       "      <td>1064351</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>us</td>\n",
       "      <td>5aAx2yezTd8zXrkmtKl66Z</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>Closer</td>\n",
       "      <td>The Chainsmokers</td>\n",
       "      <td>1010492</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>us</td>\n",
       "      <td>7BKLCZ1jbUBVqRi2FVlTVw</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>Black Beatles</td>\n",
       "      <td>Rae Sremmurd</td>\n",
       "      <td>874289</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>us</td>\n",
       "      <td>6fujklziTHa8uoM5OQSfIo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>195</td>\n",
       "      <td>196</td>\n",
       "      <td>Ignition - Remix</td>\n",
       "      <td>R. Kelly</td>\n",
       "      <td>144377</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>us</td>\n",
       "      <td>5dNfHmqgr128gMY2tc5CeJ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>196</td>\n",
       "      <td>197</td>\n",
       "      <td>679 (feat. Remy Boyz)</td>\n",
       "      <td>Fetty Wap</td>\n",
       "      <td>143877</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>us</td>\n",
       "      <td>5NQbUaeTEOGdD6hHcre0dZ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>197</td>\n",
       "      <td>198</td>\n",
       "      <td>I Don't Fuck With You</td>\n",
       "      <td>Big Sean</td>\n",
       "      <td>143847</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>us</td>\n",
       "      <td>4w7yrP4RAeeyhfG9nJqQvS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>198</td>\n",
       "      <td>199</td>\n",
       "      <td>You Can't Hurry Love - 2016 Remaster</td>\n",
       "      <td>Phil Collins</td>\n",
       "      <td>143813</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>us</td>\n",
       "      <td>4YwbSZaYeYja8Umyt222Qf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>199</td>\n",
       "      <td>200</td>\n",
       "      <td>For Free (feat. Drake)</td>\n",
       "      <td>DJ Khaled</td>\n",
       "      <td>143765</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>us</td>\n",
       "      <td>5TqBAOIBe0bQA0spNdcG3k</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>200 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Position                            Track Name            Artist  \\\n",
       "0           1   Bad and Boujee (feat. Lil Uzi Vert)             Migos   \n",
       "1           2                             Fake Love             Drake   \n",
       "2           3                               Starboy        The Weeknd   \n",
       "3           4                                Closer  The Chainsmokers   \n",
       "4           5                         Black Beatles      Rae Sremmurd   \n",
       "..        ...                                   ...               ...   \n",
       "195       196                      Ignition - Remix          R. Kelly   \n",
       "196       197                 679 (feat. Remy Boyz)         Fetty Wap   \n",
       "197       198                 I Don't Fuck With You          Big Sean   \n",
       "198       199  You Can't Hurry Love - 2016 Remaster      Phil Collins   \n",
       "199       200                For Free (feat. Drake)         DJ Khaled   \n",
       "\n",
       "     Streams        date region              spotify_id  \n",
       "0    1371493  2017-01-01     us  4Km5HrUvYTaSUfiSGPJeQR  \n",
       "1    1180074  2017-01-01     us  343YBumqHu19cGoGARUTsd  \n",
       "2    1064351  2017-01-01     us  5aAx2yezTd8zXrkmtKl66Z  \n",
       "3    1010492  2017-01-01     us  7BKLCZ1jbUBVqRi2FVlTVw  \n",
       "4     874289  2017-01-01     us  6fujklziTHa8uoM5OQSfIo  \n",
       "..       ...         ...    ...                     ...  \n",
       "195   144377  2017-01-01     us  5dNfHmqgr128gMY2tc5CeJ  \n",
       "196   143877  2017-01-01     us  5NQbUaeTEOGdD6hHcre0dZ  \n",
       "197   143847  2017-01-01     us  4w7yrP4RAeeyhfG9nJqQvS  \n",
       "198   143813  2017-01-01     us  4YwbSZaYeYja8Umyt222Qf  \n",
       "199   143765  2017-01-01     us  5TqBAOIBe0bQA0spNdcG3k  \n",
       "\n",
       "[200 rows x 7 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"Position\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Position</th>\n",
       "      <th>Track Name</th>\n",
       "      <th>Artist</th>\n",
       "      <th>Streams</th>\n",
       "      <th>date</th>\n",
       "      <th>region</th>\n",
       "      <th>spotify_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>29795</td>\n",
       "      <td>196</td>\n",
       "      <td>Wyclef Jean</td>\n",
       "      <td>Young Thug</td>\n",
       "      <td>158272</td>\n",
       "      <td>2017-05-29</td>\n",
       "      <td>us</td>\n",
       "      <td>55OdqrG8WLmsYyY1jijD9b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29796</td>\n",
       "      <td>197</td>\n",
       "      <td>Reminder</td>\n",
       "      <td>The Weeknd</td>\n",
       "      <td>158031</td>\n",
       "      <td>2017-05-29</td>\n",
       "      <td>us</td>\n",
       "      <td>37F0uwRSrdzkBiuj0D5UHI</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29797</td>\n",
       "      <td>198</td>\n",
       "      <td>FEAR.</td>\n",
       "      <td>Kendrick Lamar</td>\n",
       "      <td>157812</td>\n",
       "      <td>2017-05-29</td>\n",
       "      <td>us</td>\n",
       "      <td>23luOrEVHMfoX0AhfbQuS6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29798</td>\n",
       "      <td>199</td>\n",
       "      <td>The Weekend</td>\n",
       "      <td>Brantley Gilbert</td>\n",
       "      <td>157627</td>\n",
       "      <td>2017-05-29</td>\n",
       "      <td>us</td>\n",
       "      <td>48tAdjG4ng2JfECa7OQujk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29799</td>\n",
       "      <td>200</td>\n",
       "      <td>Candy Paint</td>\n",
       "      <td>Post Malone</td>\n",
       "      <td>157460</td>\n",
       "      <td>2017-05-29</td>\n",
       "      <td>us</td>\n",
       "      <td>4wdZdkEugVVorAfzqRqC6N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29800</td>\n",
       "      <td>Position</td>\n",
       "      <td>Track Name</td>\n",
       "      <td>Artist</td>\n",
       "      <td>Streams</td>\n",
       "      <td>date</td>\n",
       "      <td>region</td>\n",
       "      <td>spotify_id</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29801</td>\n",
       "      <td>1</td>\n",
       "      <td>Bad and Boujee (feat. Lil Uzi Vert)</td>\n",
       "      <td>Migos</td>\n",
       "      <td>1371493</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>us</td>\n",
       "      <td>4Km5HrUvYTaSUfiSGPJeQR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29802</td>\n",
       "      <td>2</td>\n",
       "      <td>Fake Love</td>\n",
       "      <td>Drake</td>\n",
       "      <td>1180074</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>us</td>\n",
       "      <td>343YBumqHu19cGoGARUTsd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29803</td>\n",
       "      <td>3</td>\n",
       "      <td>Starboy</td>\n",
       "      <td>The Weeknd</td>\n",
       "      <td>1064351</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>us</td>\n",
       "      <td>5aAx2yezTd8zXrkmtKl66Z</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29804</td>\n",
       "      <td>4</td>\n",
       "      <td>Closer</td>\n",
       "      <td>The Chainsmokers</td>\n",
       "      <td>1010492</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>us</td>\n",
       "      <td>7BKLCZ1jbUBVqRi2FVlTVw</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Position                           Track Name            Artist  \\\n",
       "29795       196                          Wyclef Jean        Young Thug   \n",
       "29796       197                             Reminder        The Weeknd   \n",
       "29797       198                                FEAR.    Kendrick Lamar   \n",
       "29798       199                          The Weekend  Brantley Gilbert   \n",
       "29799       200                          Candy Paint       Post Malone   \n",
       "29800  Position                           Track Name            Artist   \n",
       "29801         1  Bad and Boujee (feat. Lil Uzi Vert)             Migos   \n",
       "29802         2                            Fake Love             Drake   \n",
       "29803         3                              Starboy        The Weeknd   \n",
       "29804         4                               Closer  The Chainsmokers   \n",
       "\n",
       "       Streams        date  region              spotify_id  \n",
       "29795   158272  2017-05-29      us  55OdqrG8WLmsYyY1jijD9b  \n",
       "29796   158031  2017-05-29      us  37F0uwRSrdzkBiuj0D5UHI  \n",
       "29797   157812  2017-05-29      us  23luOrEVHMfoX0AhfbQuS6  \n",
       "29798   157627  2017-05-29      us  48tAdjG4ng2JfECa7OQujk  \n",
       "29799   157460  2017-05-29      us  4wdZdkEugVVorAfzqRqC6N  \n",
       "29800  Streams        date  region              spotify_id  \n",
       "29801  1371493  2017-01-01      us  4Km5HrUvYTaSUfiSGPJeQR  \n",
       "29802  1180074  2017-01-01      us  343YBumqHu19cGoGARUTsd  \n",
       "29803  1064351  2017-01-01      us  5aAx2yezTd8zXrkmtKl66Z  \n",
       "29804  1010492  2017-01-01      us  7BKLCZ1jbUBVqRi2FVlTVw  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[29795:29805]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for chunk in "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}