fetch_pdfs_toScript.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports and command line arguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "usage: __main__.py [-h] [-pmids PMIDS] [-pmf PMF] [-out OUT] [-errors ERRORS]\n",
      "                   [-maxRetries MAXRETRIES]\n",
      "__main__.py: error: unrecognized arguments: -f /run/user/1050/jupyter/kernel-5d8b80b6-ba5b-4c6d-b971-7eaa855800df.json\n"
     ]
    },
    {
     "ename": "SystemExit",
     "evalue": "2",
     "output_type": "error",
     "traceback": [
      "An exception has occurred, use %tb to see the full traceback.\n",
      "\u001b[0;31mSystemExit\u001b[0m\u001b[0;31m:\u001b[0m 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/frazer01/home/bill/software/anaconda/lib/python2.7/site-packages/IPython/core/interactiveshell.py:2890: UserWarning: To exit: use 'exit', 'quit', or Ctrl-D.\n",
      "  warn(\"To exit: use 'exit', 'quit', or Ctrl-D.\", stacklevel=1)\n"
     ]
    }
   ],
   "source": [
    "parser=argparse.ArgumentParser()\n",
    "parser._optionals.title = \"Flag Arguments\"\n",
    "parser.add_argument('-pmids',help=\"Comma separated list of pmids to fetch. Must include -pmids or -pmf.\", default='%#$')\n",
    "parser.add_argument('-pmf',help=\"File with pmids to fetch inside, one pmid per line. Optionally, the file can be a tsv with a second column of names to save each pmid's article with (without '.pdf' at the end). Must include -pmids or -pmf\", default='%#$')\n",
    "parser.add_argument('-out',help=\"Output directory for fetched articles.  Default: fetched_pdfs\", default=\"fetched_pdfs\")\n",
    "parser.add_argument('-errors',help=\"Output file path for pmids which failed to fetch.  Default: unfetched_pmids.tsv\", default=\"unfetched_pmids.tsv\")\n",
    "parser.add_argument('-maxRetries',help=\"Change max number of retries per article on an error 104.  Default: 3\", default=3,type=int)\n",
    "args = vars(parser.parse_args())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "#debugging\n",
    "#List of pmids and how they should fetch correctly (to make sure new fetchers dont break old code)\n",
    "#NEJM -- 25176136\n",
    "#Science Direct -- 25282519\n",
    "#Oxford Academics -- 26030325\n",
    "#Future Medicine -- 28589772\n",
    " \n",
    "# args={'pmids':'26030325',\n",
    "#       'pmf':'%#$',\n",
    "#       'out':'fetched_pdfs',\n",
    "#       'maxRetries':3,\n",
    "#        'errors':'unfetched_pmids.tsv'\n",
    "#       }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "if len(sys.argv)==1:\n",
    "    parser.print_help(sys.stderr)\n",
    "    exit(1)\n",
    "if args['pmids']=='%#$' and args['pmf']=='%#$':\n",
    "    print (\"Error: Either -pmids or -pmf must be used.  Exiting.\")\n",
    "    exit(1)\n",
    "if args['pmids']!='%#$' and args['pmf']!='%#$':\n",
    "    print (\"Error: -pmids and -pmf cannot be used together.  Ignoring -pmf argument\")\n",
    "    args['pmf']='%#$'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import urllib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists(args['out']):\n",
    "    print( \"Output directory of {0} did not exist.  Created the directory.\".format(args['out']))\n",
    "    os.mkdir(args['out'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Debug space.  Clear before commit"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getMainUrl(url):\n",
    "    return \"/\".join(url.split(\"/\")[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def savePdfFromUrl(pdfUrl,directory,name,headers):\n",
    "    t=requests.get(pdfUrl,headers=headers,allow_redirects=True)\n",
    "    with open('{0}/{1}.pdf'.format(directory,name), 'wb') as f:\n",
    "        f.write(t.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fetch(pmid,finders,name,headers,errorPmids):\n",
    "    \n",
    "    uri = \"http://eutils.ncbi.nlm.nih.gov/entrez/eutils/elink.fcgi?dbfrom=pubmed&id={0}&retmode=ref&cmd=prlinks\".format(pmid)\n",
    "    success = False\n",
    "    dontTry=False\n",
    "    if os.path.exists(\"{0}/{1}.pdf\".format(args['out'],pmid)): # bypass finders if pdf reprint already stored locally\n",
    "        print (\"** Reprint #{0} already downloaded and in folder; skipping.\".format(pmid))\n",
    "        return\n",
    "    else:\n",
    "        #first, download the html from the page that is on the other side of the pubmed API\n",
    "        req=requests.get(uri,headers=headers)\n",
    "        if 'ovid' in req.url:\n",
    "            print (\" ** Reprint {0} cannot be fetched as ovid is not supported by the requests package.\".format(pmid))\n",
    "            errorPmids.write(\"{}\\t{}\\n\".format(pmid,name))\n",
    "            dontTry=True\n",
    "            success=True\n",
    "        soup=BeautifulSoup(req.content,'lxml')\n",
    "#         return soup\n",
    "        # loop through all finders until it finds one that return the pdf reprint\n",
    "        if not dontTry:\n",
    "            for finder in finders:\n",
    "                print (\"Trying {0}\".format(finder))\n",
    "                pdfUrl = eval(finder)(req,soup,headers)\n",
    "                if type(pdfUrl)!=type(None):\n",
    "                    savePdfFromUrl(pdfUrl,args['out'],name,headers)\n",
    "                    success = True\n",
    "                    print (\"** fetching of reprint {0} succeeded\".format(pmid))\n",
    "                    break\n",
    "       \n",
    "        if not success:\n",
    "            print (\"** Reprint {0} could not be fetched with the current finders.\".format(pmid))\n",
    "            errorPmids.write(\"{}\\t{}\\n\".format(pmid,name))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Finders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def acsPublications(req,soup,headers):\n",
    "    possibleLinks=[x for x in soup.find_all('a') if type(x.get('title'))==str and ('high-res pdf' in x.get('title').lower() or 'low-res pdf' in x.get('title').lower())]\n",
    "    \n",
    "    if len(possibleLinks)>0:\n",
    "        print (\"** fetching reprint using the 'acsPublications' finder...\")\n",
    "        pdfUrl=getMainUrl(req.url)+possibleLinks[0].get('href')\n",
    "        return pdfUrl\n",
    "    \n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def direct_pdf_link(req,soup,headers): #if anyone has a PMID that direct links, I can debug this better\n",
    "    \n",
    "    if req.content[-4:]=='.pdf':\n",
    "        print (\"** fetching reprint using the 'direct pdf link' finder...\")\n",
    "        pdfUrl=req.content\n",
    "        return pdfUrl\n",
    "    \n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def futureMedicine(req,soup,headers):\n",
    "    possibleLinks=soup.find_all('a',attrs={'href':re.compile(\"/doi/pdf\")})\n",
    "    if len(possibleLinks)>0:\n",
    "        print (\"** fetching reprint using the 'future medicine' finder...\")\n",
    "        pdfUrl=getMainUrl(req.url)+possibleLinks[0].get('href')\n",
    "        return pdfUrl\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def genericCitationLabelled(req,soup,headers): #if anyone has CSH access, I can check this.  Also, a PMID on CSH would help debugging\n",
    "    \n",
    "    possibleLinks=soup.find_all('meta',attrs={'name':'citation_pdf_url'})\n",
    "    if len(possibleLinks)>0:\n",
    "        print (\"** fetching reprint using the 'generic citation labelled' finder...\")\n",
    "        pdfUrl=possibleLinks[0].get('content')\n",
    "        return pdfUrl\n",
    "    return None\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def nejm(req,soup,headers):\n",
    "    possibleLinks=[x for x in soup.find_all('a') if type(x.get('data-download-type'))==str and (x.get('data-download-type').lower()=='article pdf')]\n",
    "        \n",
    "    if len(possibleLinks)>0:\n",
    "        print (\"** fetching reprint using the 'NEJM' finder...\")\n",
    "        pdfUrl=getMainUrl(req.url)+possibleLinks[0].get('href')\n",
    "        return pdfUrl\n",
    "    \n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pubmed_central_v1(req,soup,headers):\n",
    "    possibleLinks=soup.find_all('a',re.compile('pdf'))\n",
    "    \n",
    "    possibleLinks=[x for x in possibleLinks if 'epdf' not in x.get('title').lower()] #this allows the pubmed_central finder to also work for wiley\n",
    "    \n",
    "    if len(possibleLinks)>0:\n",
    "        print (\"** fetching reprint using the 'pubmed central' finder...\")\n",
    "        pdfUrl=getMainUrl(req.url)+possibleLinks[0].get('href')\n",
    "        return pdfUrl\n",
    "    \n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pubmed_central_v2(req,soup,headers):\n",
    "    possibleLinks=soup.find_all('a',attrs={'href':re.compile('/pmc/articles')})\n",
    "        \n",
    "    if len(possibleLinks)>0:\n",
    "        print (\"** fetching reprint using the 'pubmed central' finder...\")\n",
    "        pdfUrl=\"https://www.ncbi.nlm.nih.gov/{}\".format(possibleLinks[0].get('href'))\n",
    "        return pdfUrl\n",
    "    \n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def science_direct(req,soup,headers):\n",
    "    newUri=urllib.parse.unquote(soup.find_all('input')[0].get('value'))\n",
    "    req=requests.get(newUri,allow_redirects=True,headers=headers)\n",
    "    soup=BeautifulSoup(req.content,'lxml')\n",
    "\n",
    "    possibleLinks=soup.find_all('meta',attrs={'name':'citation_pdf_url'})\n",
    "    \n",
    "    \n",
    "    \n",
    "    if len(possibleLinks)>0:\n",
    "        print (\"** fetching reprint using the 'science_direct' finder...\")\n",
    "        req=requests.get(possibleLinks[0].get('content'),headers=headers)\n",
    "        soup=BeautifulSoup(req.content,'lxml')\n",
    "        pdfUrl=soup.find_all('a')[0].get('href')\n",
    "        return pdfUrl\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def uchicagoPress(req,soup,headers):\n",
    "    possibleLinks=[x for x in soup.find_all('a') if type(x.get('href'))==str and 'pdf' in x.get('href') and '.edu/doi/' in x.get('href')]    \n",
    "    if len(possibleLinks)>0:\n",
    "        print (\"** fetching reprint using the 'uchicagoPress' finder...\")\n",
    "        pdfUrl=getMainUrl(req.url)+possibleLinks[0].get('href')\n",
    "        return pdfUrl\n",
    "    \n",
    "    return None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Main"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "finders=[\n",
    "         'genericCitationLabelled',\n",
    "         'pubmed_central_v2',\n",
    "         'acsPublications',\n",
    "         'uchicagoPress',\n",
    "         'nejm',\n",
    "         'futureMedicine',\n",
    "         'science_direct',\n",
    "         'direct_pdf_link',\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/elink.fcgi?dbfrom=pubmed&id=30898248&retmode=ref&cmd=prlinks\n",
      "Trying to fetch pmid 30898248\n",
      "Trying genericCitationLabelled\n",
      "Trying pubmed_central_v2\n",
      "Trying acsPublications\n",
      "Trying uchicagoPress\n",
      "Trying nejm\n",
      "Trying futureMedicine\n",
      "Trying science_direct\n",
      "** fetching of reprint 30898248 failed from error 'NoneType' object has no attribute 'split'\n"
     ]
    }
   ],
   "source": [
    "headers = requests.utils.default_headers()\n",
    "headers['User-Agent'] = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'\n",
    "\n",
    "if args['pmids']!='%#$':\n",
    "    pmids=args['pmids'].split(\",\")\n",
    "    names=pmids\n",
    "else:\n",
    "    pmids=[line.strip().split() for line in open(args['pmf'])]\n",
    "    if len(pmids[0])==1:\n",
    "        pmids=[x[0] for x in pmids]\n",
    "        names=pmids\n",
    "    else:\n",
    "        names=[x[1] for x in pmids]\n",
    "        pmids=[x[0] for x in pmids]\n",
    "\n",
    "with open(args['errors'],'w+') as errorPmids:\n",
    "    for pmid,name in zip(pmids,names):\n",
    "        print (\"Trying to fetch pmid {0}\".format(pmid))\n",
    "        retriesSoFar=0\n",
    "        while retriesSoFar<args['maxRetries']:\n",
    "            try:\n",
    "                soup=fetch(pmid,finders,name,headers,errorPmids)\n",
    "                retriesSoFar=args['maxRetries']\n",
    "            except requests.ConnectionError as e:\n",
    "                if '104' in str(e) or 'BadStatusLine' in str(e):\n",
    "                    retriesSoFar+=1\n",
    "                    if retriesSoFar<args['maxRetries']:\n",
    "                        print (\"** fetching of reprint {0} failed from error {1}, retrying\".format(pmid,e))\n",
    "                    else:\n",
    "                        print (\"** fetching of reprint {0} failed from error {1}\".format(pmid,e))\n",
    "                        errorPmids.write(\"{}\\t{}\\n\".format(pmid,name))\n",
    "                else:\n",
    "                    print (\"** fetching of reprint {0} failed from error {1}\".format(pmid,e))\n",
    "                    retriesSoFar=args['maxRetries']\n",
    "                    errorPmids.write(\"{}\\t{}\\n\".format(pmid,name))\n",
    "            except Exception as e:\n",
    "                print (\"** fetching of reprint {0} failed from error {1}\".format(pmid,e))\n",
    "                retriesSoFar=args['maxRetries']\n",
    "                errorPmids.write(\"{}\\t{}\\n\".format(pmid,name))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test cases for when adding a new finder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trying to fetch pmid 24985776\n",
      "Trying genericCitationLabelled\n",
      "Trying pubmed_central_v2\n",
      "Trying acsPublications\n",
      "Trying uchicagoPress\n",
      "Trying nejm\n",
      "Trying futureMedicine\n",
      "Trying science_direct\n",
      "** fetching reprint using the 'science_direct' finder...\n",
      "** fetching of reprint 24985776 succeeded\n"
     ]
    }
   ],
   "source": [
    "# headers = requests.utils.default_headers()\n",
    "# headers['User-Agent'] = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'\n",
    "\n",
    "# #NEJM, Science Direct, Oxford Academics, Future Medicine, Pubmed Central, Science Direct\n",
    "# pmids=['25176136','25282519','26030325','28589772','28543980', '24985776']\n",
    "# names=pmids\n",
    "\n",
    "# with open(args['errors'],'w+') as errorPmids:\n",
    "#     for pmid,name in zip(pmids,names):\n",
    "#         print (\"Trying to fetch pmid {0}\".format(pmid))\n",
    "#         retriesSoFar=0\n",
    "#         while retriesSoFar<args['maxRetries']:\n",
    "#             try:\n",
    "#                 soup=fetch(pmid,finders,name,headers,errorPmids)\n",
    "#                 retriesSoFar=args['maxRetries']\n",
    "#             except requests.ConnectionError as e:\n",
    "#                 if '104' in str(e):\n",
    "#                     retriesSoFar+=1\n",
    "#                     if retriesSoFar<args['maxRetries']:\n",
    "#                         print \"** fetching of reprint {0} failed from error {1}, retrying\".format(pmid,e)\n",
    "#                     else:\n",
    "#                         print \"** fetching of reprint {0} failed from error {1}\".format(pmid,e)\n",
    "#                 else:\n",
    "#                     print \"** fetching of reprint {0} failed from error {1}\".format(pmid,e)\n",
    "#                     retriesSoFar=args['maxRetries']\n",
    "#             except Exception as e:\n",
    "#                 print \"** fetching of reprint {0} failed from error {1}\".format(pmid,e)\n",
    "#                 retriesSoFar=args['maxRetries']"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}