Step_3_Add_citation_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### About\n",
    "This notebook will retrieve citation counts and other metrics from Dimensions.  If data has been loaded from WoS, you can choose to use WoS citation counts ('TC' or 'Z9') instead and therefore skip this step. Dimensions tend to have better coverage of citation counts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------------\n",
      "Step 3:  Add citation data\n",
      "2018-02-16 14:02:20.901055\n",
      "------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "print('------------------------------------------------------')\n",
    "print('Step 3:  Add citation data')\n",
    "from datetime import datetime as dt\n",
    "t_start = dt.now()\n",
    "print(t_start)\n",
    "print('------------------------------------------------------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adding citation data for  7710  articles\n"
     ]
    }
   ],
   "source": [
    "# import dependencies\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import pickle\n",
    "from bs4 import BeautifulSoup as bs\n",
    "# for querying scopus\n",
    "import requests\n",
    "import json\n",
    "\n",
    "# inputs\n",
    "from config import Config as c\n",
    "dois_pkl = c.dois_pkl\n",
    "dois = pickle.load(open(dois_pkl,'rb'))\n",
    "print('Adding citation data for ',len(dois),' articles')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "working_data = c.working_data\n",
    "df = pd.read_csv(working_data, index_col=0)\n",
    "filepaths_pkl = c.filepaths_pkl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading Dimensions data from cache\n"
     ]
    }
   ],
   "source": [
    "dim_data_p = 'data/dimensions_metrics.json'\n",
    "dim_failures_p = 'data/dimensions_failures.json'\n",
    "print('Reading Dimensions data from cache')\n",
    "try:\n",
    "    with open(dim_data_p,'rb') as f:\n",
    "        dim_data = json.loads(f.read())\n",
    "    successes = [x for x in dim_data.keys()]\n",
    "\n",
    "except:\n",
    "    print('Cache not found.')\n",
    "    print('Creating new cache.')\n",
    "    dim_data = {}\n",
    "    with open(dim_data_p,'wb') as f:\n",
    "        f.write(b'{}') \n",
    "    successes=[]\n",
    "        \n",
    "try:\n",
    "    with open(dim_failures_p,'rb') as f:\n",
    "        failures = json.loads(f.read())\n",
    "\n",
    "except:\n",
    "    failures = []\n",
    "    with open(dim_failures_p,'wb') as f:\n",
    "        f.write(b'[]') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "#define api call\n",
    "def get_dimensions(doi, dim_data):\n",
    "    try:\n",
    "        out = dim_data[doi]\n",
    "    except:\n",
    "        base= 'http://metrics-api.dimensions.ai/doi/'\n",
    "        r = requests.get(base+doi)\n",
    "        out = r.json()\n",
    "        dim_data[doi] = out\n",
    "    return out, dim_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# example\n",
    "# get_dimensions('10.1007/s00401-012-1028-y',{})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Retrieving data from Dimension Metrics API\n",
      "Data retrieved for:  0 / 7710  articles\n",
      "Data retrieved for:  100 / 7710  articles\n",
      "Data retrieved for:  200 / 7710  articles\n",
      "Data retrieved for:  300 / 7710  articles\n",
      "Data retrieved for:  400 / 7710  articles\n",
      "Data retrieved for:  500 / 7710  articles\n",
      "Data retrieved for:  600 / 7710  articles\n",
      "Data retrieved for:  700 / 7710  articles\n",
      "Data retrieved for:  800 / 7710  articles\n",
      "Data retrieved for:  900 / 7710  articles\n",
      "Data retrieved for:  1000 / 7710  articles\n",
      "Data retrieved for:  1100 / 7710  articles\n",
      "Data retrieved for:  1200 / 7710  articles\n",
      "Data retrieved for:  1300 / 7710  articles\n",
      "Data retrieved for:  1400 / 7710  articles\n",
      "Data retrieved for:  1500 / 7710  articles\n",
      "Data retrieved for:  1600 / 7710  articles\n",
      "Data retrieved for:  1700 / 7710  articles\n",
      "Data retrieved for:  1800 / 7710  articles\n",
      "Data retrieved for:  1900 / 7710  articles\n",
      "Data retrieved for:  2000 / 7710  articles\n",
      "Error. Recording data for 10.1523/JNEUROSCI.3579-13.014 as NaN.\n",
      "Data retrieved for:  2100 / 7710  articles\n",
      "Data retrieved for:  2200 / 7710  articles\n",
      "Error. Recording data for 10.1523/0NEUROSCI.3235-13.2014 as NaN.\n",
      "Data retrieved for:  2300 / 7710  articles\n",
      "Data retrieved for:  2400 / 7710  articles\n",
      "Data retrieved for:  2500 / 7710  articles\n",
      "Error. Recording data for 10.1523/JNEUROSCI.3308-13-2013 as NaN.\n",
      "Data retrieved for:  2600 / 7710  articles\n",
      "Data retrieved for:  2700 / 7710  articles\n",
      "Data retrieved for:  2800 / 7710  articles\n",
      "Data retrieved for:  2900 / 7710  articles\n",
      "Data retrieved for:  3000 / 7710  articles\n",
      "Data retrieved for:  3100 / 7710  articles\n",
      "Data retrieved for:  3200 / 7710  articles\n",
      "Error. Recording data for 10.1523/JNEUROSC.5629-12.2013 as NaN.\n",
      "Data retrieved for:  3300 / 7710  articles\n",
      "Data retrieved for:  3400 / 7710  articles\n",
      "Data retrieved for:  3500 / 7710  articles\n",
      "Data retrieved for:  3600 / 7710  articles\n",
      "Data retrieved for:  3700 / 7710  articles\n",
      "Data retrieved for:  3800 / 7710  articles\n",
      "Data retrieved for:  3900 / 7710  articles\n",
      "Data retrieved for:  4000 / 7710  articles\n",
      "Data retrieved for:  4100 / 7710  articles\n",
      "Data retrieved for:  4200 / 7710  articles\n",
      "Error. Recording data for 10.1016/S1474-4422(14)70138-X as NaN.\n",
      "Error. Recording data for 10.1016/S1474-4421(14)70068-7 as NaN.\n",
      "Error. Recording data for 10.1016/S1474-4421(14)70051-1 as NaN.\n",
      "Data retrieved for:  4300 / 7710  articles\n",
      "Error. Recording data for 10.1016/S1474-4422(13)700374 as NaN.\n",
      "Error. Recording data for 10.1016/S1424-4422(12)20269-7 as NaN.\n",
      "Data retrieved for:  4400 / 7710  articles\n",
      "Data retrieved for:  4500 / 7710  articles\n",
      "Data retrieved for:  4600 / 7710  articles\n",
      "Data retrieved for:  4700 / 7710  articles\n",
      "Error. Recording data for 10.1177/13524585124.51944 as NaN.\n",
      "Data retrieved for:  4800 / 7710  articles\n",
      "Data retrieved for:  4900 / 7710  articles\n",
      "Data retrieved for:  5000 / 7710  articles\n",
      "Error. Recording data for 10.1016/j.neuroimage2014.04.078 as NaN.\n",
      "Error. Recording data for 10.1016/j.neuroimage2014.05.013 as NaN.\n",
      "Error. Recording data for 10.1016/j.neuroimage2014.04.063 as NaN.\n",
      "Error. Recording data for 10.1016/j.neuroimage2014.05.014 as NaN.\n",
      "Error. Recording data for 10.1016/j.neruoimage.2014.04.065 as NaN.\n",
      "Data retrieved for:  5100 / 7710  articles\n",
      "Error. Recording data for 10.1016/j.neuroimage2014.04.059 as NaN.\n",
      "Data retrieved for:  5200 / 7710  articles\n",
      "Error. Recording data for 10.1016/j.neuroimage2014.02.013 as NaN.\n",
      "Data retrieved for:  5300 / 7710  articles\n",
      "Error. Recording data for 10.1016/j.neuroimage2013.11.053 as NaN.\n",
      "Error. Recording data for 10.1016/j.neuroimage2013.12.040 as NaN.\n",
      "Error. Recording data for 10.1016/j.neuroimage2013.12.003 as NaN.\n",
      "Error. Recording data for 10.1016/j.neuroimage2013.12.010 as NaN.\n",
      "Data retrieved for:  5400 / 7710  articles\n",
      "Error. Recording data for 10.1016/j.neuroimage2013.11.012 as NaN.\n",
      "Error. Recording data for 10.1016/j.neuroimage2013.11.049 as NaN.\n",
      "Data retrieved for:  5500 / 7710  articles\n",
      "Error. Recording data for 10.1016/j.neuroimage2013.07.085 as NaN.\n",
      "Error. Recording data for 10.1016/j.neuroimage2013.07.082 as NaN.\n",
      "Error. Recording data for 10.1016/j.neuroimage2013.09.053 as NaN.\n",
      "Error. Recording data for 10.1016/ineuroimage2013.10.007 as NaN.\n",
      "Error. Recording data for 10.1016/j.neuroimage2013.10.006 as NaN.\n",
      "Error. Recording data for 10.1016/j.neuroimage2013.07.078 as NaN.\n",
      "Error. Recording data for 10.1016/ineuroimage.2013.07.023 as NaN.\n",
      "Data retrieved for:  5600 / 7710  articles\n",
      "Error. Recording data for 10.1016/j.neuroimage2013.08.024 as NaN.\n",
      "Error. Recording data for 10.1016/ineuroimage.2013.08.035 as NaN.\n",
      "Error. Recording data for 10.1016/j.neuroimage2013.09.010 as NaN.\n",
      "Error. Recording data for 10.1016/j.neuroimage2013.09.038 as NaN.\n",
      "Error. Recording data for 10.1016/neuroimage.2013.09.040 as NaN.\n",
      "Error. Recording data for 10.1016/j.neuroimage2013.09.032 as NaN.\n",
      "Data retrieved for:  5700 / 7710  articles\n",
      "Error. Recording data for 10.1016/j.neuroimage2013.09.029 as NaN.\n",
      "Error. Recording data for 10.1016/ineuroimage.2013.07.058 as NaN.\n",
      "Data retrieved for:  5800 / 7710  articles\n",
      "Error. Recording data for 10.1016/j.neuroimage.013.07.054 as NaN.\n",
      "Error. Recording data for 10.1016/ineuroimage.2013.07.070 as NaN.\n",
      "Error. Recording data for 10.1016/Ineuroimage.2013.07.074 as NaN.\n",
      "Data retrieved for:  5900 / 7710  articles\n",
      "Data retrieved for:  6000 / 7710  articles\n",
      "Data retrieved for:  6100 / 7710  articles\n",
      "Data retrieved for:  6200 / 7710  articles\n",
      "Error. Recording data for 10.1016/j.neuroimage.012.12.047 as NaN.\n",
      "Data retrieved for:  6300 / 7710  articles\n",
      "Error. Recording data for 10.1016/j.neuroimage,2012.11.030 as NaN.\n",
      "Error. Recording data for 10.1016/j.neuroim;ge.2012.11.015 as NaN.\n",
      "Data retrieved for:  6400 / 7710  articles\n",
      "Data retrieved for:  6500 / 7710  articles\n",
      "Data retrieved for:  6600 / 7710  articles\n",
      "Data retrieved for:  6700 / 7710  articles\n",
      "Data retrieved for:  6800 / 7710  articles\n",
      "Data retrieved for:  6900 / 7710  articles\n",
      "Data retrieved for:  7000 / 7710  articles\n",
      "Data retrieved for:  7100 / 7710  articles\n",
      "Data retrieved for:  7200 / 7710  articles\n",
      "Data retrieved for:  7300 / 7710  articles\n",
      "Data retrieved for:  7400 / 7710  articles\n",
      "Error. Recording data for 10.1016/1.neuron.2013.08.013 as NaN.\n",
      "Data retrieved for:  7500 / 7710  articles\n",
      "Data retrieved for:  7600 / 7710  articles\n",
      "Data retrieved for:  7700 / 7710  articles\n"
     ]
    }
   ],
   "source": [
    "# d, below is here to allow easily building a dataframe with dois as index.  Not really needed.\n",
    "# this also means that try/except bits below are a little odd\n",
    "\n",
    "d = {'field_citation_ratio': [],\n",
    " 'highly_cited_1': [],\n",
    " 'highly_cited_10': [],\n",
    " 'highly_cited_5': [],\n",
    " 'recent_citations': [],\n",
    " 'relative_citation_ratio': [],\n",
    " 'times_cited': []}\n",
    "\n",
    "i=0\n",
    "print('Retrieving data from Dimension Metrics API')\n",
    "for doi in dois:\n",
    "    if i%100==0:\n",
    "        print('Data retrieved for: ',i,'/',len(dois),' articles')\n",
    "    try:\n",
    "        doi_dim_data, dim_data = get_dimensions(doi, dim_data)\n",
    "        for dim in d:\n",
    "            d[dim].append(doi_dim_data[dim]) # note that just 1 error here leads to all NaNs for 1 article\n",
    "    except:\n",
    "        print('Error. Recording data for {} as NaN.'.format(doi))\n",
    "        for dim in d:\n",
    "            d[dim].append(np.nan)\n",
    "        if doi not in failures:\n",
    "            failures.append(doi)\n",
    "    else:\n",
    "        pass\n",
    "    i+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total no of failures:  45\n"
     ]
    }
   ],
   "source": [
    "print('Total no of failures: ', len(failures))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing Dimensions data to file.\n"
     ]
    }
   ],
   "source": [
    "# write data to files\n",
    "print('Writing Dimensions data to file.')\n",
    "with open(dim_data_p,'w+') as f:\n",
    "    f.write(json.dumps(dim_data))\n",
    "with open(dim_failures_p,'w+') as f:\n",
    "    f.write(json.dumps(failures))    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Construct dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "# add dois col\n",
    "d['DOI'] = dois\n",
    "# add links col\n",
    "links = [r'http://dx.doi.org/'+doi for doi in dois]\n",
    "d['Link'] = links\n",
    "# reformat dict as a dataframe\n",
    "df_dims = pd.DataFrame(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# if we're replacing the Dimensions data, we need to drop it from the existing df\n",
    "try:\n",
    "    df = df.drop(['field_citation_ratio','highly_cited_1',\n",
    "                    'highly_cited_10','highly_cited_5','recent_citations',\n",
    "                    'relative_citation_ratio','times_cited'], axis=1)\n",
    "except:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Combining Dimensions citation metrics with existing dataset\n"
     ]
    }
   ],
   "source": [
    "print('Combining Dimensions citation metrics with existing dataset')\n",
    "rows_before = df.shape[0]\n",
    "df = df.merge(right=df_dims, left_on='DI',right_on='DOI',how='left',copy=False)\n",
    "rows_after = df.shape[0]\n",
    "if rows_before!=rows_after:\n",
    "    print('Warning - merging Dimensions data is adding/losing rows')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# pick a column to be the 'citations' column.  Choose from: \n",
    "# TC  (WoS Core collection)\n",
    "# Z9  (all WoS data)\n",
    "# times_cited (Dimensions citation count (which seems to be best))\n",
    "# note that you can also choose other columns from the Dimensions data, like RCR, FCR etc\n",
    "\n",
    "df['Citations'] = df.times_cited"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# write to file\n",
    "df_dims.to_csv('data/Dimensions_data.csv')\n",
    "df.to_csv(c.working_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done in  0:00:04.900712\n"
     ]
    }
   ],
   "source": [
    "print(\"Done in \",dt.now()-t_start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}