examples/Concepts_in_SYNERGY.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0c87888d",
   "metadata": {},
   "source": [
    "# How concepts in SYNERGY are computed\n",
    "\n",
    "Concepts, also known as topics, are attached to each OpenAlex record. They are generated with an machine leanring model. You can indetify at least 3 different types of concepts per dataset in SYNERGY:\n",
    "\n",
    "- The main (level 0) concept of the publication record\n",
    "- The average main (level 0) concept of all records in the dataset (the search)\n",
    "- The average main (level 0) concept of all included records in the dataset\n",
    "\n",
    "See the table at the bottom of this notebook for the topic per dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bfb47b7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import synergy_dataset as sd\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "26232888",
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to standardize concept data\n",
    "\n",
    "def flatten_concepts(ds, level=0):\n",
    "    result = []\n",
    "    for openalex_id, record in ds.to_dict([\"concepts\"]).items():\n",
    "\n",
    "        d = {\"openalex_id\": openalex_id,\n",
    "             \"label_included\": record[\"label_included\"]}\n",
    "\n",
    "        for c in record[\"concepts\"]:\n",
    "            if c[\"level\"] == level:\n",
    "                d[c[\"display_name\"]] = c[\"score\"]\n",
    "\n",
    "        result.append(d)\n",
    "    \n",
    "    return pd.DataFrame(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b1799ef0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>openalex_id</th>\n",
       "      <th>label_included</th>\n",
       "      <th>Medicine</th>\n",
       "      <th>Chemistry</th>\n",
       "      <th>Physics</th>\n",
       "      <th>Biology</th>\n",
       "      <th>Psychology</th>\n",
       "      <th>Economics</th>\n",
       "      <th>Materials science</th>\n",
       "      <th>Philosophy</th>\n",
       "      <th>...</th>\n",
       "      <th>Political science</th>\n",
       "      <th>History</th>\n",
       "      <th>Computer science</th>\n",
       "      <th>Sociology</th>\n",
       "      <th>Mathematics</th>\n",
       "      <th>Geology</th>\n",
       "      <th>Business</th>\n",
       "      <th>Art</th>\n",
       "      <th>Geography</th>\n",
       "      <th>Environmental science</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">Appenzeller-Herzog_2019</th>\n",
       "      <th>0</th>\n",
       "      <td>https://openalex.org/W1971651073</td>\n",
       "      <td>0</td>\n",
       "      <td>0.56412965</td>\n",
       "      <td>0.09127462</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://openalex.org/W2060266518</td>\n",
       "      <td>0</td>\n",
       "      <td>0.4624714</td>\n",
       "      <td>0.31808412</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.36406675</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://openalex.org/W1985329347</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.8244176</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.10137832</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://openalex.org/W2033425122</td>\n",
       "      <td>0</td>\n",
       "      <td>0.5881644</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://openalex.org/W2174281073</td>\n",
       "      <td>0</td>\n",
       "      <td>0.41348755</td>\n",
       "      <td>0.35551745</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">Wolters_2018</th>\n",
       "      <th>4275</th>\n",
       "      <td>https://openalex.org/W1995096921</td>\n",
       "      <td>0</td>\n",
       "      <td>0.91154534</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4276</th>\n",
       "      <td>https://openalex.org/W2114922669</td>\n",
       "      <td>0</td>\n",
       "      <td>0.95886254</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4277</th>\n",
       "      <td>https://openalex.org/W1657669688</td>\n",
       "      <td>0</td>\n",
       "      <td>0.96129704</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4278</th>\n",
       "      <td>https://openalex.org/W1994490975</td>\n",
       "      <td>0</td>\n",
       "      <td>0.9699614</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4279</th>\n",
       "      <td>https://openalex.org/W1504586396</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8246919</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>169288 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   openalex_id   \n",
       "Appenzeller-Herzog_2019 0     https://openalex.org/W1971651073  \\\n",
       "                        1     https://openalex.org/W2060266518   \n",
       "                        2     https://openalex.org/W1985329347   \n",
       "                        3     https://openalex.org/W2033425122   \n",
       "                        4     https://openalex.org/W2174281073   \n",
       "...                                                        ...   \n",
       "Wolters_2018            4275  https://openalex.org/W1995096921   \n",
       "                        4276  https://openalex.org/W2114922669   \n",
       "                        4277  https://openalex.org/W1657669688   \n",
       "                        4278  https://openalex.org/W1994490975   \n",
       "                        4279  https://openalex.org/W1504586396   \n",
       "\n",
       "                              label_included    Medicine   Chemistry Physics   \n",
       "Appenzeller-Herzog_2019 0                  0  0.56412965  0.09127462     0.0  \\\n",
       "                        1                  0   0.4624714  0.31808412     NaN   \n",
       "                        2                  0         0.0   0.8244176     NaN   \n",
       "                        3                  0   0.5881644         NaN     NaN   \n",
       "                        4                  0  0.41348755  0.35551745     NaN   \n",
       "...                                      ...         ...         ...     ...   \n",
       "Wolters_2018            4275               0  0.91154534         NaN     NaN   \n",
       "                        4276               0  0.95886254         NaN     NaN   \n",
       "                        4277               0  0.96129704         NaN     NaN   \n",
       "                        4278               0   0.9699614         NaN     NaN   \n",
       "                        4279               0   0.8246919         NaN     NaN   \n",
       "\n",
       "                                 Biology Psychology Economics   \n",
       "Appenzeller-Herzog_2019 0            NaN        NaN       NaN  \\\n",
       "                        1     0.36406675        NaN       NaN   \n",
       "                        2     0.10137832        NaN       NaN   \n",
       "                        3            NaN        NaN       NaN   \n",
       "                        4            NaN        NaN       NaN   \n",
       "...                                  ...        ...       ...   \n",
       "Wolters_2018            4275         NaN        NaN       NaN   \n",
       "                        4276         NaN        NaN       NaN   \n",
       "                        4277         NaN        NaN       NaN   \n",
       "                        4278         NaN        NaN       NaN   \n",
       "                        4279         NaN        NaN       NaN   \n",
       "\n",
       "                             Materials science Philosophy  ...   \n",
       "Appenzeller-Herzog_2019 0                  NaN        NaN  ...  \\\n",
       "                        1                  NaN        NaN  ...   \n",
       "                        2                  NaN        NaN  ...   \n",
       "                        3                  NaN        NaN  ...   \n",
       "                        4                  NaN        NaN  ...   \n",
       "...                                        ...        ...  ...   \n",
       "Wolters_2018            4275               NaN        NaN  ...   \n",
       "                        4276               NaN        NaN  ...   \n",
       "                        4277               NaN        NaN  ...   \n",
       "                        4278               NaN        NaN  ...   \n",
       "                        4279               NaN        NaN  ...   \n",
       "\n",
       "                             Political science History Computer science   \n",
       "Appenzeller-Herzog_2019 0                  NaN     NaN              NaN  \\\n",
       "                        1                  NaN     NaN              NaN   \n",
       "                        2                  NaN     NaN              NaN   \n",
       "                        3                  NaN     NaN              NaN   \n",
       "                        4                  NaN     NaN              NaN   \n",
       "...                                        ...     ...              ...   \n",
       "Wolters_2018            4275               NaN     NaN              NaN   \n",
       "                        4276               NaN     NaN              NaN   \n",
       "                        4277               NaN     NaN              NaN   \n",
       "                        4278               NaN     NaN              NaN   \n",
       "                        4279               NaN     NaN              NaN   \n",
       "\n",
       "                             Sociology Mathematics Geology Business  Art   \n",
       "Appenzeller-Herzog_2019 0          NaN         NaN     NaN      NaN  NaN  \\\n",
       "                        1          NaN         NaN     NaN      NaN  NaN   \n",
       "                        2          NaN         NaN     NaN      NaN  NaN   \n",
       "                        3          NaN         NaN     NaN      NaN  NaN   \n",
       "                        4          NaN         NaN     NaN      NaN  NaN   \n",
       "...                                ...         ...     ...      ...  ...   \n",
       "Wolters_2018            4275       NaN         NaN     NaN      NaN  NaN   \n",
       "                        4276       NaN         NaN     NaN      NaN  NaN   \n",
       "                        4277       NaN         NaN     NaN      NaN  NaN   \n",
       "                        4278       NaN         NaN     NaN      NaN  NaN   \n",
       "                        4279       NaN         NaN     NaN      NaN  NaN   \n",
       "\n",
       "                             Geography Environmental science  \n",
       "Appenzeller-Herzog_2019 0          NaN                   NaN  \n",
       "                        1          NaN                   NaN  \n",
       "                        2          NaN                   NaN  \n",
       "                        3          NaN                   NaN  \n",
       "                        4          NaN                   NaN  \n",
       "...                                ...                   ...  \n",
       "Wolters_2018            4275       NaN                   NaN  \n",
       "                        4276       NaN                   NaN  \n",
       "                        4277       NaN                   NaN  \n",
       "                        4278       NaN                   NaN  \n",
       "                        4279       NaN                   NaN  \n",
       "\n",
       "[169288 rows x 21 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keys = [d.name for d in sd.iter_datasets()]\n",
    "df = pd.concat([flatten_concepts(d) for d in sd.iter_datasets()], keys=keys)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85068a93",
   "metadata": {},
   "source": [
    "The following table shows the average concept scores for all records in the dataset. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6db00fb3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Medicine</th>\n",
       "      <th>Chemistry</th>\n",
       "      <th>Physics</th>\n",
       "      <th>Biology</th>\n",
       "      <th>Psychology</th>\n",
       "      <th>Economics</th>\n",
       "      <th>Materials science</th>\n",
       "      <th>Philosophy</th>\n",
       "      <th>Engineering</th>\n",
       "      <th>Political science</th>\n",
       "      <th>History</th>\n",
       "      <th>Computer science</th>\n",
       "      <th>Sociology</th>\n",
       "      <th>Mathematics</th>\n",
       "      <th>Geology</th>\n",
       "      <th>Business</th>\n",
       "      <th>Art</th>\n",
       "      <th>Geography</th>\n",
       "      <th>Environmental science</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Appenzeller-Herzog_2019</th>\n",
       "      <td>0.588982</td>\n",
       "      <td>0.103742</td>\n",
       "      <td>0.002754</td>\n",
       "      <td>0.054124</td>\n",
       "      <td>0.030139</td>\n",
       "      <td>0.000246</td>\n",
       "      <td>0.004020</td>\n",
       "      <td>0.006154</td>\n",
       "      <td>0.000616</td>\n",
       "      <td>0.002405</td>\n",
       "      <td>0.002357</td>\n",
       "      <td>0.006770</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000370</td>\n",
       "      <td>0.000309</td>\n",
       "      <td>0.001908</td>\n",
       "      <td>0.000138</td>\n",
       "      <td>0.001154</td>\n",
       "      <td>0.001068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bos_2018</th>\n",
       "      <td>0.565684</td>\n",
       "      <td>0.008519</td>\n",
       "      <td>0.004312</td>\n",
       "      <td>0.028526</td>\n",
       "      <td>0.202546</td>\n",
       "      <td>0.000056</td>\n",
       "      <td>0.000249</td>\n",
       "      <td>0.000376</td>\n",
       "      <td>0.000072</td>\n",
       "      <td>0.000096</td>\n",
       "      <td>0.000118</td>\n",
       "      <td>0.015857</td>\n",
       "      <td>0.000069</td>\n",
       "      <td>0.004413</td>\n",
       "      <td>0.000068</td>\n",
       "      <td>0.000109</td>\n",
       "      <td>0.000021</td>\n",
       "      <td>0.000077</td>\n",
       "      <td>0.000067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Brouwer_2019</th>\n",
       "      <td>0.332740</td>\n",
       "      <td>0.006930</td>\n",
       "      <td>0.000627</td>\n",
       "      <td>0.011639</td>\n",
       "      <td>0.451345</td>\n",
       "      <td>0.001146</td>\n",
       "      <td>0.000212</td>\n",
       "      <td>0.004143</td>\n",
       "      <td>0.000596</td>\n",
       "      <td>0.002774</td>\n",
       "      <td>0.001253</td>\n",
       "      <td>0.006507</td>\n",
       "      <td>0.004893</td>\n",
       "      <td>0.001216</td>\n",
       "      <td>0.000184</td>\n",
       "      <td>0.001035</td>\n",
       "      <td>0.001908</td>\n",
       "      <td>0.001189</td>\n",
       "      <td>0.000237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Chou_2003</th>\n",
       "      <td>0.865703</td>\n",
       "      <td>0.005159</td>\n",
       "      <td>0.000069</td>\n",
       "      <td>0.002794</td>\n",
       "      <td>0.011938</td>\n",
       "      <td>0.000102</td>\n",
       "      <td>0.000122</td>\n",
       "      <td>0.000174</td>\n",
       "      <td>0.000111</td>\n",
       "      <td>0.001076</td>\n",
       "      <td>0.000052</td>\n",
       "      <td>0.002323</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000330</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000125</td>\n",
       "      <td>0.000033</td>\n",
       "      <td>0.000090</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Chou_2004</th>\n",
       "      <td>0.760822</td>\n",
       "      <td>0.009780</td>\n",
       "      <td>0.000445</td>\n",
       "      <td>0.010023</td>\n",
       "      <td>0.056741</td>\n",
       "      <td>0.000225</td>\n",
       "      <td>0.000560</td>\n",
       "      <td>0.000095</td>\n",
       "      <td>0.000701</td>\n",
       "      <td>0.001352</td>\n",
       "      <td>0.000218</td>\n",
       "      <td>0.004468</td>\n",
       "      <td>0.000153</td>\n",
       "      <td>0.000146</td>\n",
       "      <td>0.000068</td>\n",
       "      <td>0.001643</td>\n",
       "      <td>0.000033</td>\n",
       "      <td>0.000079</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Donners_2021</th>\n",
       "      <td>0.663195</td>\n",
       "      <td>0.053750</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.027470</td>\n",
       "      <td>0.002286</td>\n",
       "      <td>0.000324</td>\n",
       "      <td>0.001471</td>\n",
       "      <td>0.002270</td>\n",
       "      <td>0.002108</td>\n",
       "      <td>0.009966</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.016382</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001673</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.005976</td>\n",
       "      <td>0.002458</td>\n",
       "      <td>0.000747</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hall_2012</th>\n",
       "      <td>0.003415</td>\n",
       "      <td>0.001369</td>\n",
       "      <td>0.032076</td>\n",
       "      <td>0.001220</td>\n",
       "      <td>0.001738</td>\n",
       "      <td>0.000374</td>\n",
       "      <td>0.020137</td>\n",
       "      <td>0.000058</td>\n",
       "      <td>0.139658</td>\n",
       "      <td>0.000253</td>\n",
       "      <td>0.000091</td>\n",
       "      <td>0.599865</td>\n",
       "      <td>0.000096</td>\n",
       "      <td>0.055082</td>\n",
       "      <td>0.012524</td>\n",
       "      <td>0.002743</td>\n",
       "      <td>0.000054</td>\n",
       "      <td>0.001584</td>\n",
       "      <td>0.004120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Jeyaraman_2020</th>\n",
       "      <td>0.723767</td>\n",
       "      <td>0.060434</td>\n",
       "      <td>0.000291</td>\n",
       "      <td>0.041080</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.055064</td>\n",
       "      <td>0.000452</td>\n",
       "      <td>0.001324</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.003419</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000392</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000179</td>\n",
       "      <td>0.000108</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Leenaars_2019</th>\n",
       "      <td>0.233326</td>\n",
       "      <td>0.420204</td>\n",
       "      <td>0.000652</td>\n",
       "      <td>0.161313</td>\n",
       "      <td>0.064596</td>\n",
       "      <td>0.000053</td>\n",
       "      <td>0.001316</td>\n",
       "      <td>0.000404</td>\n",
       "      <td>0.000146</td>\n",
       "      <td>0.001013</td>\n",
       "      <td>0.000138</td>\n",
       "      <td>0.002701</td>\n",
       "      <td>0.000075</td>\n",
       "      <td>0.000171</td>\n",
       "      <td>0.000025</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>0.000077</td>\n",
       "      <td>0.000055</td>\n",
       "      <td>0.000050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Leenaars_2020</th>\n",
       "      <td>0.839694</td>\n",
       "      <td>0.017233</td>\n",
       "      <td>0.000281</td>\n",
       "      <td>0.023396</td>\n",
       "      <td>0.000667</td>\n",
       "      <td>0.000301</td>\n",
       "      <td>0.001126</td>\n",
       "      <td>0.000470</td>\n",
       "      <td>0.000206</td>\n",
       "      <td>0.000479</td>\n",
       "      <td>0.000014</td>\n",
       "      <td>0.003330</td>\n",
       "      <td>0.000029</td>\n",
       "      <td>0.001062</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000488</td>\n",
       "      <td>0.000247</td>\n",
       "      <td>0.000094</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Meijboom_2021</th>\n",
       "      <td>0.751780</td>\n",
       "      <td>0.005847</td>\n",
       "      <td>0.000922</td>\n",
       "      <td>0.010856</td>\n",
       "      <td>0.003104</td>\n",
       "      <td>0.009470</td>\n",
       "      <td>0.000908</td>\n",
       "      <td>0.003851</td>\n",
       "      <td>0.003206</td>\n",
       "      <td>0.015895</td>\n",
       "      <td>0.000344</td>\n",
       "      <td>0.026981</td>\n",
       "      <td>0.000786</td>\n",
       "      <td>0.003590</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.059610</td>\n",
       "      <td>0.000231</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Menon_2022</th>\n",
       "      <td>0.675263</td>\n",
       "      <td>0.008930</td>\n",
       "      <td>0.000492</td>\n",
       "      <td>0.031988</td>\n",
       "      <td>0.086502</td>\n",
       "      <td>0.001591</td>\n",
       "      <td>0.003524</td>\n",
       "      <td>0.000590</td>\n",
       "      <td>0.004443</td>\n",
       "      <td>0.010007</td>\n",
       "      <td>0.000071</td>\n",
       "      <td>0.016922</td>\n",
       "      <td>0.002459</td>\n",
       "      <td>0.001514</td>\n",
       "      <td>0.000176</td>\n",
       "      <td>0.008145</td>\n",
       "      <td>0.000479</td>\n",
       "      <td>0.006455</td>\n",
       "      <td>0.007136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Moran_2021</th>\n",
       "      <td>0.172927</td>\n",
       "      <td>0.019691</td>\n",
       "      <td>0.002657</td>\n",
       "      <td>0.360793</td>\n",
       "      <td>0.191964</td>\n",
       "      <td>0.006161</td>\n",
       "      <td>0.000850</td>\n",
       "      <td>0.001209</td>\n",
       "      <td>0.001594</td>\n",
       "      <td>0.003316</td>\n",
       "      <td>0.000249</td>\n",
       "      <td>0.016642</td>\n",
       "      <td>0.003379</td>\n",
       "      <td>0.006788</td>\n",
       "      <td>0.002283</td>\n",
       "      <td>0.005363</td>\n",
       "      <td>0.000747</td>\n",
       "      <td>0.029374</td>\n",
       "      <td>0.012471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Muthu_2021</th>\n",
       "      <td>0.919395</td>\n",
       "      <td>0.001366</td>\n",
       "      <td>0.000361</td>\n",
       "      <td>0.000862</td>\n",
       "      <td>0.002161</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001703</td>\n",
       "      <td>0.000059</td>\n",
       "      <td>0.000543</td>\n",
       "      <td>0.000118</td>\n",
       "      <td>0.000062</td>\n",
       "      <td>0.001575</td>\n",
       "      <td>0.000029</td>\n",
       "      <td>0.001079</td>\n",
       "      <td>0.000067</td>\n",
       "      <td>0.000048</td>\n",
       "      <td>0.000012</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Nelson_2002</th>\n",
       "      <td>0.794984</td>\n",
       "      <td>0.014132</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.009409</td>\n",
       "      <td>0.042286</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.003390</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Oud_2018</th>\n",
       "      <td>0.263017</td>\n",
       "      <td>0.000188</td>\n",
       "      <td>0.000075</td>\n",
       "      <td>0.001155</td>\n",
       "      <td>0.536209</td>\n",
       "      <td>0.000634</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.004119</td>\n",
       "      <td>0.000360</td>\n",
       "      <td>0.003439</td>\n",
       "      <td>0.000467</td>\n",
       "      <td>0.004379</td>\n",
       "      <td>0.000843</td>\n",
       "      <td>0.001024</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000141</td>\n",
       "      <td>0.000650</td>\n",
       "      <td>0.000346</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Radjenovic_2013</th>\n",
       "      <td>0.002892</td>\n",
       "      <td>0.001334</td>\n",
       "      <td>0.013897</td>\n",
       "      <td>0.001137</td>\n",
       "      <td>0.002358</td>\n",
       "      <td>0.000509</td>\n",
       "      <td>0.009470</td>\n",
       "      <td>0.000147</td>\n",
       "      <td>0.105367</td>\n",
       "      <td>0.000324</td>\n",
       "      <td>0.000141</td>\n",
       "      <td>0.658504</td>\n",
       "      <td>0.000124</td>\n",
       "      <td>0.032756</td>\n",
       "      <td>0.005237</td>\n",
       "      <td>0.007445</td>\n",
       "      <td>0.000118</td>\n",
       "      <td>0.001316</td>\n",
       "      <td>0.003877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sep_2021</th>\n",
       "      <td>0.142033</td>\n",
       "      <td>0.040661</td>\n",
       "      <td>0.006123</td>\n",
       "      <td>0.139960</td>\n",
       "      <td>0.491806</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000365</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000402</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.079383</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.005571</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002218</td>\n",
       "      <td>0.001093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Smid_2020</th>\n",
       "      <td>0.065957</td>\n",
       "      <td>0.003701</td>\n",
       "      <td>0.008060</td>\n",
       "      <td>0.013416</td>\n",
       "      <td>0.176623</td>\n",
       "      <td>0.024159</td>\n",
       "      <td>0.000471</td>\n",
       "      <td>0.002025</td>\n",
       "      <td>0.006444</td>\n",
       "      <td>0.017591</td>\n",
       "      <td>0.000391</td>\n",
       "      <td>0.247746</td>\n",
       "      <td>0.014569</td>\n",
       "      <td>0.277334</td>\n",
       "      <td>0.001526</td>\n",
       "      <td>0.010506</td>\n",
       "      <td>0.000279</td>\n",
       "      <td>0.009510</td>\n",
       "      <td>0.006835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Walker_2018</th>\n",
       "      <td>0.374658</td>\n",
       "      <td>0.072131</td>\n",
       "      <td>0.007581</td>\n",
       "      <td>0.242093</td>\n",
       "      <td>0.062255</td>\n",
       "      <td>0.004979</td>\n",
       "      <td>0.012615</td>\n",
       "      <td>0.001947</td>\n",
       "      <td>0.002788</td>\n",
       "      <td>0.009339</td>\n",
       "      <td>0.003266</td>\n",
       "      <td>0.013151</td>\n",
       "      <td>0.011774</td>\n",
       "      <td>0.005582</td>\n",
       "      <td>0.000623</td>\n",
       "      <td>0.004298</td>\n",
       "      <td>0.001245</td>\n",
       "      <td>0.006425</td>\n",
       "      <td>0.002598</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wassenaar_2017</th>\n",
       "      <td>0.128569</td>\n",
       "      <td>0.265142</td>\n",
       "      <td>0.001793</td>\n",
       "      <td>0.107622</td>\n",
       "      <td>0.003896</td>\n",
       "      <td>0.000204</td>\n",
       "      <td>0.306068</td>\n",
       "      <td>0.000110</td>\n",
       "      <td>0.002743</td>\n",
       "      <td>0.000589</td>\n",
       "      <td>0.000135</td>\n",
       "      <td>0.005615</td>\n",
       "      <td>0.000076</td>\n",
       "      <td>0.004512</td>\n",
       "      <td>0.002840</td>\n",
       "      <td>0.002640</td>\n",
       "      <td>0.000363</td>\n",
       "      <td>0.000437</td>\n",
       "      <td>0.018556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wolters_2018</th>\n",
       "      <td>0.735717</td>\n",
       "      <td>0.014164</td>\n",
       "      <td>0.000699</td>\n",
       "      <td>0.043370</td>\n",
       "      <td>0.038102</td>\n",
       "      <td>0.000411</td>\n",
       "      <td>0.000367</td>\n",
       "      <td>0.000740</td>\n",
       "      <td>0.000752</td>\n",
       "      <td>0.001789</td>\n",
       "      <td>0.000838</td>\n",
       "      <td>0.007482</td>\n",
       "      <td>0.000813</td>\n",
       "      <td>0.001353</td>\n",
       "      <td>0.000136</td>\n",
       "      <td>0.002363</td>\n",
       "      <td>0.000413</td>\n",
       "      <td>0.001670</td>\n",
       "      <td>0.000499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>van_Dis_2020</th>\n",
       "      <td>0.384957</td>\n",
       "      <td>0.006082</td>\n",
       "      <td>0.000838</td>\n",
       "      <td>0.003165</td>\n",
       "      <td>0.421250</td>\n",
       "      <td>0.000263</td>\n",
       "      <td>0.002460</td>\n",
       "      <td>0.000490</td>\n",
       "      <td>0.000425</td>\n",
       "      <td>0.001231</td>\n",
       "      <td>0.000159</td>\n",
       "      <td>0.005751</td>\n",
       "      <td>0.000511</td>\n",
       "      <td>0.001618</td>\n",
       "      <td>0.000097</td>\n",
       "      <td>0.000506</td>\n",
       "      <td>0.000302</td>\n",
       "      <td>0.000343</td>\n",
       "      <td>0.000381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>van_de_Schoot_2018</th>\n",
       "      <td>0.303815</td>\n",
       "      <td>0.004313</td>\n",
       "      <td>0.000723</td>\n",
       "      <td>0.012298</td>\n",
       "      <td>0.454105</td>\n",
       "      <td>0.001076</td>\n",
       "      <td>0.000389</td>\n",
       "      <td>0.000920</td>\n",
       "      <td>0.002162</td>\n",
       "      <td>0.009534</td>\n",
       "      <td>0.002561</td>\n",
       "      <td>0.012151</td>\n",
       "      <td>0.006926</td>\n",
       "      <td>0.006971</td>\n",
       "      <td>0.000477</td>\n",
       "      <td>0.002066</td>\n",
       "      <td>0.000397</td>\n",
       "      <td>0.007108</td>\n",
       "      <td>0.002056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>van_der_Valk_2021</th>\n",
       "      <td>0.530890</td>\n",
       "      <td>0.020956</td>\n",
       "      <td>0.000496</td>\n",
       "      <td>0.097759</td>\n",
       "      <td>0.136118</td>\n",
       "      <td>0.001302</td>\n",
       "      <td>0.000709</td>\n",
       "      <td>0.000178</td>\n",
       "      <td>0.001031</td>\n",
       "      <td>0.002709</td>\n",
       "      <td>0.000762</td>\n",
       "      <td>0.007179</td>\n",
       "      <td>0.002435</td>\n",
       "      <td>0.000783</td>\n",
       "      <td>0.000373</td>\n",
       "      <td>0.001603</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.004380</td>\n",
       "      <td>0.001726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>van_der_Waal_2022</th>\n",
       "      <td>0.716030</td>\n",
       "      <td>0.000532</td>\n",
       "      <td>0.000065</td>\n",
       "      <td>0.016228</td>\n",
       "      <td>0.091749</td>\n",
       "      <td>0.001577</td>\n",
       "      <td>0.000061</td>\n",
       "      <td>0.000574</td>\n",
       "      <td>0.000977</td>\n",
       "      <td>0.009574</td>\n",
       "      <td>0.000156</td>\n",
       "      <td>0.021907</td>\n",
       "      <td>0.005572</td>\n",
       "      <td>0.001507</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.005600</td>\n",
       "      <td>0.000042</td>\n",
       "      <td>0.000878</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         Medicine  Chemistry   Physics   Biology  Psychology   \n",
       "Appenzeller-Herzog_2019  0.588982   0.103742  0.002754  0.054124    0.030139  \\\n",
       "Bos_2018                 0.565684   0.008519  0.004312  0.028526    0.202546   \n",
       "Brouwer_2019             0.332740   0.006930  0.000627  0.011639    0.451345   \n",
       "Chou_2003                0.865703   0.005159  0.000069  0.002794    0.011938   \n",
       "Chou_2004                0.760822   0.009780  0.000445  0.010023    0.056741   \n",
       "Donners_2021             0.663195   0.053750  0.000000  0.027470    0.002286   \n",
       "Hall_2012                0.003415   0.001369  0.032076  0.001220    0.001738   \n",
       "Jeyaraman_2020           0.723767   0.060434  0.000291  0.041080    0.000000   \n",
       "Leenaars_2019            0.233326   0.420204  0.000652  0.161313    0.064596   \n",
       "Leenaars_2020            0.839694   0.017233  0.000281  0.023396    0.000667   \n",
       "Meijboom_2021            0.751780   0.005847  0.000922  0.010856    0.003104   \n",
       "Menon_2022               0.675263   0.008930  0.000492  0.031988    0.086502   \n",
       "Moran_2021               0.172927   0.019691  0.002657  0.360793    0.191964   \n",
       "Muthu_2021               0.919395   0.001366  0.000361  0.000862    0.002161   \n",
       "Nelson_2002              0.794984   0.014132  0.000000  0.009409    0.042286   \n",
       "Oud_2018                 0.263017   0.000188  0.000075  0.001155    0.536209   \n",
       "Radjenovic_2013          0.002892   0.001334  0.013897  0.001137    0.002358   \n",
       "Sep_2021                 0.142033   0.040661  0.006123  0.139960    0.491806   \n",
       "Smid_2020                0.065957   0.003701  0.008060  0.013416    0.176623   \n",
       "Walker_2018              0.374658   0.072131  0.007581  0.242093    0.062255   \n",
       "Wassenaar_2017           0.128569   0.265142  0.001793  0.107622    0.003896   \n",
       "Wolters_2018             0.735717   0.014164  0.000699  0.043370    0.038102   \n",
       "van_Dis_2020             0.384957   0.006082  0.000838  0.003165    0.421250   \n",
       "van_de_Schoot_2018       0.303815   0.004313  0.000723  0.012298    0.454105   \n",
       "van_der_Valk_2021        0.530890   0.020956  0.000496  0.097759    0.136118   \n",
       "van_der_Waal_2022        0.716030   0.000532  0.000065  0.016228    0.091749   \n",
       "\n",
       "                         Economics  Materials science  Philosophy   \n",
       "Appenzeller-Herzog_2019   0.000246           0.004020    0.006154  \\\n",
       "Bos_2018                  0.000056           0.000249    0.000376   \n",
       "Brouwer_2019              0.001146           0.000212    0.004143   \n",
       "Chou_2003                 0.000102           0.000122    0.000174   \n",
       "Chou_2004                 0.000225           0.000560    0.000095   \n",
       "Donners_2021              0.000324           0.001471    0.002270   \n",
       "Hall_2012                 0.000374           0.020137    0.000058   \n",
       "Jeyaraman_2020            0.000000           0.055064    0.000452   \n",
       "Leenaars_2019             0.000053           0.001316    0.000404   \n",
       "Leenaars_2020             0.000301           0.001126    0.000470   \n",
       "Meijboom_2021             0.009470           0.000908    0.003851   \n",
       "Menon_2022                0.001591           0.003524    0.000590   \n",
       "Moran_2021                0.006161           0.000850    0.001209   \n",
       "Muthu_2021                0.000000           0.001703    0.000059   \n",
       "Nelson_2002               0.000000           0.000000    0.000000   \n",
       "Oud_2018                  0.000634           0.000000    0.004119   \n",
       "Radjenovic_2013           0.000509           0.009470    0.000147   \n",
       "Sep_2021                  0.000000           0.000365    0.000000   \n",
       "Smid_2020                 0.024159           0.000471    0.002025   \n",
       "Walker_2018               0.004979           0.012615    0.001947   \n",
       "Wassenaar_2017            0.000204           0.306068    0.000110   \n",
       "Wolters_2018              0.000411           0.000367    0.000740   \n",
       "van_Dis_2020              0.000263           0.002460    0.000490   \n",
       "van_de_Schoot_2018        0.001076           0.000389    0.000920   \n",
       "van_der_Valk_2021         0.001302           0.000709    0.000178   \n",
       "van_der_Waal_2022         0.001577           0.000061    0.000574   \n",
       "\n",
       "                         Engineering  Political science   History   \n",
       "Appenzeller-Herzog_2019     0.000616           0.002405  0.002357  \\\n",
       "Bos_2018                    0.000072           0.000096  0.000118   \n",
       "Brouwer_2019                0.000596           0.002774  0.001253   \n",
       "Chou_2003                   0.000111           0.001076  0.000052   \n",
       "Chou_2004                   0.000701           0.001352  0.000218   \n",
       "Donners_2021                0.002108           0.009966  0.000000   \n",
       "Hall_2012                   0.139658           0.000253  0.000091   \n",
       "Jeyaraman_2020              0.001324           0.000000  0.000000   \n",
       "Leenaars_2019               0.000146           0.001013  0.000138   \n",
       "Leenaars_2020               0.000206           0.000479  0.000014   \n",
       "Meijboom_2021               0.003206           0.015895  0.000344   \n",
       "Menon_2022                  0.004443           0.010007  0.000071   \n",
       "Moran_2021                  0.001594           0.003316  0.000249   \n",
       "Muthu_2021                  0.000543           0.000118  0.000062   \n",
       "Nelson_2002                 0.000000           0.000000  0.000000   \n",
       "Oud_2018                    0.000360           0.003439  0.000467   \n",
       "Radjenovic_2013             0.105367           0.000324  0.000141   \n",
       "Sep_2021                    0.000402           0.000000  0.000000   \n",
       "Smid_2020                   0.006444           0.017591  0.000391   \n",
       "Walker_2018                 0.002788           0.009339  0.003266   \n",
       "Wassenaar_2017              0.002743           0.000589  0.000135   \n",
       "Wolters_2018                0.000752           0.001789  0.000838   \n",
       "van_Dis_2020                0.000425           0.001231  0.000159   \n",
       "van_de_Schoot_2018          0.002162           0.009534  0.002561   \n",
       "van_der_Valk_2021           0.001031           0.002709  0.000762   \n",
       "van_der_Waal_2022           0.000977           0.009574  0.000156   \n",
       "\n",
       "                         Computer science  Sociology  Mathematics   Geology   \n",
       "Appenzeller-Herzog_2019          0.006770   0.000000     0.000370  0.000309  \\\n",
       "Bos_2018                         0.015857   0.000069     0.004413  0.000068   \n",
       "Brouwer_2019                     0.006507   0.004893     0.001216  0.000184   \n",
       "Chou_2003                        0.002323   0.000000     0.000330  0.000000   \n",
       "Chou_2004                        0.004468   0.000153     0.000146  0.000068   \n",
       "Donners_2021                     0.016382   0.000000     0.001673  0.000000   \n",
       "Hall_2012                        0.599865   0.000096     0.055082  0.012524   \n",
       "Jeyaraman_2020                   0.003419   0.000000     0.000392  0.000000   \n",
       "Leenaars_2019                    0.002701   0.000075     0.000171  0.000025   \n",
       "Leenaars_2020                    0.003330   0.000029     0.001062  0.000000   \n",
       "Meijboom_2021                    0.026981   0.000786     0.003590  0.000000   \n",
       "Menon_2022                       0.016922   0.002459     0.001514  0.000176   \n",
       "Moran_2021                       0.016642   0.003379     0.006788  0.002283   \n",
       "Muthu_2021                       0.001575   0.000029     0.001079  0.000067   \n",
       "Nelson_2002                      0.003390   0.000000     0.000000  0.000000   \n",
       "Oud_2018                         0.004379   0.000843     0.001024  0.000000   \n",
       "Radjenovic_2013                  0.658504   0.000124     0.032756  0.005237   \n",
       "Sep_2021                         0.079383   0.000000     0.005571  0.000000   \n",
       "Smid_2020                        0.247746   0.014569     0.277334  0.001526   \n",
       "Walker_2018                      0.013151   0.011774     0.005582  0.000623   \n",
       "Wassenaar_2017                   0.005615   0.000076     0.004512  0.002840   \n",
       "Wolters_2018                     0.007482   0.000813     0.001353  0.000136   \n",
       "van_Dis_2020                     0.005751   0.000511     0.001618  0.000097   \n",
       "van_de_Schoot_2018               0.012151   0.006926     0.006971  0.000477   \n",
       "van_der_Valk_2021                0.007179   0.002435     0.000783  0.000373   \n",
       "van_der_Waal_2022                0.021907   0.005572     0.001507  0.000000   \n",
       "\n",
       "                         Business       Art  Geography  Environmental science  \n",
       "Appenzeller-Herzog_2019  0.001908  0.000138   0.001154               0.001068  \n",
       "Bos_2018                 0.000109  0.000021   0.000077               0.000067  \n",
       "Brouwer_2019             0.001035  0.001908   0.001189               0.000237  \n",
       "Chou_2003                0.000125  0.000033   0.000090               0.000000  \n",
       "Chou_2004                0.001643  0.000033   0.000079               0.000000  \n",
       "Donners_2021             0.005976  0.002458   0.000747               0.000000  \n",
       "Hall_2012                0.002743  0.000054   0.001584               0.004120  \n",
       "Jeyaraman_2020           0.000179  0.000108   0.000000               0.000000  \n",
       "Leenaars_2019            0.000084  0.000077   0.000055               0.000050  \n",
       "Leenaars_2020            0.000488  0.000247   0.000094               0.000000  \n",
       "Meijboom_2021            0.059610  0.000231   0.000000               0.000443  \n",
       "Menon_2022               0.008145  0.000479   0.006455               0.007136  \n",
       "Moran_2021               0.005363  0.000747   0.029374               0.012471  \n",
       "Muthu_2021               0.000048  0.000012   0.000000               0.000000  \n",
       "Nelson_2002              0.000000  0.000000   0.000000               0.000000  \n",
       "Oud_2018                 0.000141  0.000650   0.000346               0.000000  \n",
       "Radjenovic_2013          0.007445  0.000118   0.001316               0.003877  \n",
       "Sep_2021                 0.000000  0.000000   0.002218               0.001093  \n",
       "Smid_2020                0.010506  0.000279   0.009510               0.006835  \n",
       "Walker_2018              0.004298  0.001245   0.006425               0.002598  \n",
       "Wassenaar_2017           0.002640  0.000363   0.000437               0.018556  \n",
       "Wolters_2018             0.002363  0.000413   0.001670               0.000499  \n",
       "van_Dis_2020             0.000506  0.000302   0.000343               0.000381  \n",
       "van_de_Schoot_2018       0.002066  0.000397   0.007108               0.002056  \n",
       "van_der_Valk_2021        0.001603  0.000000   0.004380               0.001726  \n",
       "van_der_Waal_2022        0.005600  0.000042   0.000878               0.000000  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concepts_per_dataset = df.drop([\"label_included\", \"openalex_id\"], axis=1).astype(float).fillna(0).groupby(level=0).mean()\n",
    "concepts_per_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "651cf16f",
   "metadata": {},
   "source": [
    "Same for the inclusions per dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9fff3204",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Medicine</th>\n",
       "      <th>Chemistry</th>\n",
       "      <th>Physics</th>\n",
       "      <th>Biology</th>\n",
       "      <th>Psychology</th>\n",
       "      <th>Economics</th>\n",
       "      <th>Materials science</th>\n",
       "      <th>Philosophy</th>\n",
       "      <th>Engineering</th>\n",
       "      <th>Political science</th>\n",
       "      <th>History</th>\n",
       "      <th>Computer science</th>\n",
       "      <th>Sociology</th>\n",
       "      <th>Mathematics</th>\n",
       "      <th>Geology</th>\n",
       "      <th>Business</th>\n",
       "      <th>Art</th>\n",
       "      <th>Geography</th>\n",
       "      <th>Environmental science</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Appenzeller-Herzog_2019</th>\n",
       "      <td>0.791792</td>\n",
       "      <td>0.006277</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.008704</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.004839</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.013664</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bos_2018</th>\n",
       "      <td>0.731475</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.134993</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.004561</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Brouwer_2019</th>\n",
       "      <td>0.211561</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.602484</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Chou_2003</th>\n",
       "      <td>0.847146</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.007400</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Chou_2004</th>\n",
       "      <td>0.695533</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.056843</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Donners_2021</th>\n",
       "      <td>0.817031</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hall_2012</th>\n",
       "      <td>0.000729</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000634</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.061725</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.694767</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.022457</td>\n",
       "      <td>0.014671</td>\n",
       "      <td>0.000869</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Jeyaraman_2020</th>\n",
       "      <td>0.906323</td>\n",
       "      <td>0.004976</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001607</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.004009</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Leenaars_2019</th>\n",
       "      <td>0.243685</td>\n",
       "      <td>0.245800</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.154579</td>\n",
       "      <td>0.256428</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.003783</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Leenaars_2020</th>\n",
       "      <td>0.865833</td>\n",
       "      <td>0.023698</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.008905</td>\n",
       "      <td>0.000295</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001317</td>\n",
       "      <td>0.000289</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000511</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000449</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000097</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Meijboom_2021</th>\n",
       "      <td>0.893434</td>\n",
       "      <td>0.001885</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.005540</td>\n",
       "      <td>0.003633</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002753</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Menon_2022</th>\n",
       "      <td>0.605102</td>\n",
       "      <td>0.013309</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.084069</td>\n",
       "      <td>0.059496</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002961</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000934</td>\n",
       "      <td>0.001961</td>\n",
       "      <td>0.002882</td>\n",
       "      <td>0.005709</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Moran_2021</th>\n",
       "      <td>0.214867</td>\n",
       "      <td>0.011122</td>\n",
       "      <td>0.004680</td>\n",
       "      <td>0.338177</td>\n",
       "      <td>0.130457</td>\n",
       "      <td>0.011797</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.004281</td>\n",
       "      <td>0.004192</td>\n",
       "      <td>0.011113</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.025016</td>\n",
       "      <td>0.015065</td>\n",
       "      <td>0.004918</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.027881</td>\n",
       "      <td>0.000767</td>\n",
       "      <td>0.045758</td>\n",
       "      <td>0.013757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Muthu_2021</th>\n",
       "      <td>0.950681</td>\n",
       "      <td>0.000157</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000250</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000766</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001318</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Nelson_2002</th>\n",
       "      <td>0.882701</td>\n",
       "      <td>0.011632</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001003</td>\n",
       "      <td>0.005031</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Oud_2018</th>\n",
       "      <td>0.262635</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.578095</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Radjenovic_2013</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001385</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.044402</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.715961</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.014704</td>\n",
       "      <td>0.009479</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sep_2021</th>\n",
       "      <td>0.168830</td>\n",
       "      <td>0.015908</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.134895</td>\n",
       "      <td>0.516370</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.096358</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002150</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001546</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Smid_2020</th>\n",
       "      <td>0.006124</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.100378</td>\n",
       "      <td>0.003326</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.012175</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.396219</td>\n",
       "      <td>0.009117</td>\n",
       "      <td>0.352785</td>\n",
       "      <td>0.002835</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.009769</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Walker_2018</th>\n",
       "      <td>0.228265</td>\n",
       "      <td>0.033683</td>\n",
       "      <td>0.004704</td>\n",
       "      <td>0.409869</td>\n",
       "      <td>0.085124</td>\n",
       "      <td>0.003957</td>\n",
       "      <td>0.001498</td>\n",
       "      <td>0.001726</td>\n",
       "      <td>0.001329</td>\n",
       "      <td>0.006010</td>\n",
       "      <td>0.006222</td>\n",
       "      <td>0.008706</td>\n",
       "      <td>0.011896</td>\n",
       "      <td>0.001155</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000429</td>\n",
       "      <td>0.001872</td>\n",
       "      <td>0.009046</td>\n",
       "      <td>0.001661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wassenaar_2017</th>\n",
       "      <td>0.318602</td>\n",
       "      <td>0.202573</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.310828</td>\n",
       "      <td>0.006055</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001893</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.003658</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000889</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000976</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.004943</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wolters_2018</th>\n",
       "      <td>0.799883</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.016084</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002401</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>van_Dis_2020</th>\n",
       "      <td>0.335423</td>\n",
       "      <td>0.001256</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000907</td>\n",
       "      <td>0.500181</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001041</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>van_de_Schoot_2018</th>\n",
       "      <td>0.385027</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.439764</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002325</td>\n",
       "      <td>0.003553</td>\n",
       "      <td>0.003014</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.005976</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>van_der_Valk_2021</th>\n",
       "      <td>0.553100</td>\n",
       "      <td>0.017645</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.047967</td>\n",
       "      <td>0.211837</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001950</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001074</td>\n",
       "      <td>0.004501</td>\n",
       "      <td>0.002411</td>\n",
       "      <td>0.000707</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.005417</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>van_der_Waal_2022</th>\n",
       "      <td>0.750499</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002678</td>\n",
       "      <td>0.088723</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001679</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.003656</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002142</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         Medicine  Chemistry   Physics   Biology  Psychology   \n",
       "Appenzeller-Herzog_2019  0.791792   0.006277  0.000000  0.008704    0.000000  \\\n",
       "Bos_2018                 0.731475   0.000000  0.000000  0.000000    0.134993   \n",
       "Brouwer_2019             0.211561   0.000000  0.000000  0.000000    0.602484   \n",
       "Chou_2003                0.847146   0.000000  0.000000  0.000000    0.007400   \n",
       "Chou_2004                0.695533   0.000000  0.000000  0.000000    0.056843   \n",
       "Donners_2021             0.817031   0.000000  0.000000  0.000000    0.000000   \n",
       "Hall_2012                0.000729   0.000000  0.000000  0.000634    0.000000   \n",
       "Jeyaraman_2020           0.906323   0.004976  0.000000  0.001607    0.000000   \n",
       "Leenaars_2019            0.243685   0.245800  0.000000  0.154579    0.256428   \n",
       "Leenaars_2020            0.865833   0.023698  0.000000  0.008905    0.000295   \n",
       "Meijboom_2021            0.893434   0.001885  0.000000  0.000000    0.005540   \n",
       "Menon_2022               0.605102   0.013309  0.000000  0.084069    0.059496   \n",
       "Moran_2021               0.214867   0.011122  0.004680  0.338177    0.130457   \n",
       "Muthu_2021               0.950681   0.000157  0.000000  0.000000    0.000250   \n",
       "Nelson_2002              0.882701   0.011632  0.000000  0.001003    0.005031   \n",
       "Oud_2018                 0.262635   0.000000  0.000000  0.000000    0.578095   \n",
       "Radjenovic_2013          0.000000   0.001385  0.000000  0.000000    0.000000   \n",
       "Sep_2021                 0.168830   0.015908  0.000000  0.134895    0.516370   \n",
       "Smid_2020                0.006124   0.000000  0.000000  0.000000    0.100378   \n",
       "Walker_2018              0.228265   0.033683  0.004704  0.409869    0.085124   \n",
       "Wassenaar_2017           0.318602   0.202573  0.000000  0.310828    0.006055   \n",
       "Wolters_2018             0.799883   0.000000  0.000000  0.000000    0.016084   \n",
       "van_Dis_2020             0.335423   0.001256  0.000000  0.000907    0.500181   \n",
       "van_de_Schoot_2018       0.385027   0.000000  0.000000  0.000000    0.439764   \n",
       "van_der_Valk_2021        0.553100   0.017645  0.000000  0.047967    0.211837   \n",
       "van_der_Waal_2022        0.750499   0.000000  0.000000  0.002678    0.088723   \n",
       "\n",
       "                         Economics  Materials science  Philosophy   \n",
       "Appenzeller-Herzog_2019   0.000000           0.000000    0.004839  \\\n",
       "Bos_2018                  0.000000           0.000000    0.004561   \n",
       "Brouwer_2019              0.000000           0.000000    0.000000   \n",
       "Chou_2003                 0.000000           0.000000    0.000000   \n",
       "Chou_2004                 0.000000           0.000000    0.000000   \n",
       "Donners_2021              0.000000           0.000000    0.000000   \n",
       "Hall_2012                 0.000000           0.000000    0.000000   \n",
       "Jeyaraman_2020            0.000000           0.004009    0.000000   \n",
       "Leenaars_2019             0.000000           0.000000    0.000000   \n",
       "Leenaars_2020             0.000000           0.001317    0.000289   \n",
       "Meijboom_2021             0.003633           0.000000    0.000000   \n",
       "Menon_2022                0.000000           0.000000    0.000000   \n",
       "Moran_2021                0.011797           0.000000    0.004281   \n",
       "Muthu_2021                0.000000           0.000766    0.000000   \n",
       "Nelson_2002               0.000000           0.000000    0.000000   \n",
       "Oud_2018                  0.000000           0.000000    0.000000   \n",
       "Radjenovic_2013           0.000000           0.000000    0.000000   \n",
       "Sep_2021                  0.000000           0.000000    0.000000   \n",
       "Smid_2020                 0.003326           0.000000    0.000000   \n",
       "Walker_2018               0.003957           0.001498    0.001726   \n",
       "Wassenaar_2017            0.000000           0.000000    0.001893   \n",
       "Wolters_2018              0.000000           0.000000    0.002401   \n",
       "van_Dis_2020              0.000000           0.001041    0.000000   \n",
       "van_de_Schoot_2018        0.000000           0.000000    0.000000   \n",
       "van_der_Valk_2021         0.000000           0.001950    0.000000   \n",
       "van_der_Waal_2022         0.000000           0.000000    0.000000   \n",
       "\n",
       "                         Engineering  Political science   History   \n",
       "Appenzeller-Herzog_2019     0.000000           0.000000  0.013664  \\\n",
       "Bos_2018                    0.000000           0.000000  0.000000   \n",
       "Brouwer_2019                0.000000           0.000000  0.000000   \n",
       "Chou_2003                   0.000000           0.000000  0.000000   \n",
       "Chou_2004                   0.000000           0.000000  0.000000   \n",
       "Donners_2021                0.000000           0.000000  0.000000   \n",
       "Hall_2012                   0.061725           0.000000  0.000000   \n",
       "Jeyaraman_2020              0.000000           0.000000  0.000000   \n",
       "Leenaars_2019               0.000000           0.000000  0.000000   \n",
       "Leenaars_2020               0.000000           0.000000  0.000000   \n",
       "Meijboom_2021               0.000000           0.002753  0.000000   \n",
       "Menon_2022                  0.002961           0.000000  0.000934   \n",
       "Moran_2021                  0.004192           0.011113  0.000000   \n",
       "Muthu_2021                  0.000000           0.000000  0.000000   \n",
       "Nelson_2002                 0.000000           0.000000  0.000000   \n",
       "Oud_2018                    0.000000           0.000000  0.000000   \n",
       "Radjenovic_2013             0.044402           0.000000  0.000000   \n",
       "Sep_2021                    0.000000           0.000000  0.000000   \n",
       "Smid_2020                   0.012175           0.000000  0.000000   \n",
       "Walker_2018                 0.001329           0.006010  0.006222   \n",
       "Wassenaar_2017              0.000000           0.003658  0.000000   \n",
       "Wolters_2018                0.000000           0.000000  0.000000   \n",
       "van_Dis_2020                0.000000           0.000000  0.000000   \n",
       "van_de_Schoot_2018          0.000000           0.000000  0.002325   \n",
       "van_der_Valk_2021           0.000000           0.000000  0.001074   \n",
       "van_der_Waal_2022           0.000000           0.001679  0.000000   \n",
       "\n",
       "                         Computer science  Sociology  Mathematics   Geology   \n",
       "Appenzeller-Herzog_2019          0.000000   0.000000     0.000000  0.000000  \\\n",
       "Bos_2018                         0.000000   0.000000     0.000000  0.000000   \n",
       "Brouwer_2019                     0.000000   0.000000     0.000000  0.000000   \n",
       "Chou_2003                        0.000000   0.000000     0.000000  0.000000   \n",
       "Chou_2004                        0.000000   0.000000     0.000000  0.000000   \n",
       "Donners_2021                     0.000000   0.000000     0.000000  0.000000   \n",
       "Hall_2012                        0.694767   0.000000     0.022457  0.014671   \n",
       "Jeyaraman_2020                   0.000000   0.000000     0.000000  0.000000   \n",
       "Leenaars_2019                    0.003783   0.000000     0.000000  0.000000   \n",
       "Leenaars_2020                    0.000511   0.000000     0.000449  0.000000   \n",
       "Meijboom_2021                    0.000000   0.000000     0.000000  0.000000   \n",
       "Menon_2022                       0.001961   0.002882     0.005709  0.000000   \n",
       "Moran_2021                       0.025016   0.015065     0.004918  0.000000   \n",
       "Muthu_2021                       0.001318   0.000000     0.000000  0.000000   \n",
       "Nelson_2002                      0.000000   0.000000     0.000000  0.000000   \n",
       "Oud_2018                         0.000000   0.000000     0.000000  0.000000   \n",
       "Radjenovic_2013                  0.715961   0.000000     0.014704  0.009479   \n",
       "Sep_2021                         0.096358   0.000000     0.002150  0.000000   \n",
       "Smid_2020                        0.396219   0.009117     0.352785  0.002835   \n",
       "Walker_2018                      0.008706   0.011896     0.001155  0.000000   \n",
       "Wassenaar_2017                   0.000889   0.000000     0.000976  0.000000   \n",
       "Wolters_2018                     0.000000   0.000000     0.000000  0.000000   \n",
       "van_Dis_2020                     0.000000   0.000000     0.000000  0.000000   \n",
       "van_de_Schoot_2018               0.003553   0.003014     0.000000  0.000000   \n",
       "van_der_Valk_2021                0.004501   0.002411     0.000707  0.000000   \n",
       "van_der_Waal_2022                0.000000   0.003656     0.000000  0.000000   \n",
       "\n",
       "                         Business       Art  Geography  Environmental science  \n",
       "Appenzeller-Herzog_2019  0.000000  0.000000   0.000000               0.000000  \n",
       "Bos_2018                 0.000000  0.000000   0.000000               0.000000  \n",
       "Brouwer_2019             0.000000  0.000000   0.000000               0.000000  \n",
       "Chou_2003                0.000000  0.000000   0.000000               0.000000  \n",
       "Chou_2004                0.000000  0.000000   0.000000               0.000000  \n",
       "Donners_2021             0.000000  0.000000   0.000000               0.000000  \n",
       "Hall_2012                0.000869  0.000000   0.000000               0.000000  \n",
       "Jeyaraman_2020           0.000000  0.000000   0.000000               0.000000  \n",
       "Leenaars_2019            0.000000  0.000000   0.000000               0.000000  \n",
       "Leenaars_2020            0.000097  0.000000   0.000000               0.000000  \n",
       "Meijboom_2021            0.000000  0.000000   0.000000               0.000000  \n",
       "Menon_2022               0.000000  0.000000   0.000000               0.000000  \n",
       "Moran_2021               0.027881  0.000767   0.045758               0.013757  \n",
       "Muthu_2021               0.000000  0.000000   0.000000               0.000000  \n",
       "Nelson_2002              0.000000  0.000000   0.000000               0.000000  \n",
       "Oud_2018                 0.000000  0.000000   0.000000               0.000000  \n",
       "Radjenovic_2013          0.000000  0.000000   0.000000               0.000000  \n",
       "Sep_2021                 0.000000  0.000000   0.001546               0.000000  \n",
       "Smid_2020                0.000000  0.000000   0.009769               0.000000  \n",
       "Walker_2018              0.000429  0.001872   0.009046               0.001661  \n",
       "Wassenaar_2017           0.004943  0.000000   0.000000               0.000000  \n",
       "Wolters_2018             0.000000  0.000000   0.000000               0.000000  \n",
       "van_Dis_2020             0.000000  0.000000   0.000000               0.000000  \n",
       "van_de_Schoot_2018       0.000000  0.000000   0.005976               0.000000  \n",
       "van_der_Valk_2021        0.000000  0.000000   0.005417               0.000000  \n",
       "van_der_Waal_2022        0.002142  0.000000   0.000000               0.000000  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concepts_per_dataset_incl = df[df[\"label_included\"] == 1].drop([\"label_included\", \"openalex_id\"], axis=1).astype(float).fillna(0).groupby(level=0).mean()\n",
    "concepts_per_dataset_incl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "52b97a50",
   "metadata": {},
   "outputs": [],
   "source": [
    "st = concepts_per_dataset.stack()\n",
    "st = st[st.groupby(level=0).transform(max) == st].reset_index(level=1)\n",
    "st.columns = [\"concept_search\", \"score_search\"]\n",
    "\n",
    "st_incl = concepts_per_dataset_incl.stack()\n",
    "st_incl = st_incl[st_incl.groupby(level=0).transform(max) == st_incl].reset_index(level=1)\n",
    "st_incl.columns = [\"concept_inclusions\", \"score_inclusions\"]\n",
    "\n",
    "concepts_pub = []\n",
    "for d in sd.iter_datasets():\n",
    "    r = {\"name\": d.name, \"concept_publication\": filter(lambda x: (x[\"level\"] == 0), d.metadata[\"publication\"][\"concepts\"]).__next__()[\"display_name\"]}\n",
    "    concepts_pub.append(r)\n",
    "    \n",
    "df_pub = pd.DataFrame(concepts_pub).set_index(\"name\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "78a77fb0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Appenzeller-Herzog_2019  Medicine             0.588982\n",
       "Bos_2018                 Medicine             0.565684\n",
       "                         Psychology           0.202546\n",
       "Brouwer_2019             Medicine             0.332740\n",
       "                         Psychology           0.451345\n",
       "Chou_2003                Medicine             0.865703\n",
       "Chou_2004                Medicine             0.760822\n",
       "Donners_2021             Medicine             0.663195\n",
       "Hall_2012                Computer science     0.599865\n",
       "Jeyaraman_2020           Medicine             0.723767\n",
       "Leenaars_2019            Medicine             0.233326\n",
       "                         Chemistry            0.420204\n",
       "Leenaars_2020            Medicine             0.839694\n",
       "Meijboom_2021            Medicine             0.751780\n",
       "Menon_2022               Medicine             0.675263\n",
       "Moran_2021               Biology              0.360793\n",
       "Muthu_2021               Medicine             0.919395\n",
       "Nelson_2002              Medicine             0.794984\n",
       "Oud_2018                 Medicine             0.263017\n",
       "                         Psychology           0.536209\n",
       "Radjenovic_2013          Computer science     0.658504\n",
       "Sep_2021                 Psychology           0.491806\n",
       "Smid_2020                Computer science     0.247746\n",
       "                         Mathematics          0.277334\n",
       "Walker_2018              Medicine             0.374658\n",
       "                         Biology              0.242093\n",
       "Wassenaar_2017           Chemistry            0.265142\n",
       "                         Materials science    0.306068\n",
       "Wolters_2018             Medicine             0.735717\n",
       "van_Dis_2020             Medicine             0.384957\n",
       "                         Psychology           0.421250\n",
       "van_de_Schoot_2018       Medicine             0.303815\n",
       "                         Psychology           0.454105\n",
       "van_der_Valk_2021        Medicine             0.530890\n",
       "van_der_Waal_2022        Medicine             0.716030\n",
       "dtype: float64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "st_0n = concepts_per_dataset.stack()\n",
    "st_0n[st_0n >=0.2] \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2e54f6b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Appenzeller-Herzog_2019  Medicine            0.791792\n",
       "Bos_2018                 Medicine            0.731475\n",
       "Brouwer_2019             Medicine            0.211561\n",
       "                         Psychology          0.602484\n",
       "Chou_2003                Medicine            0.847146\n",
       "Chou_2004                Medicine            0.695533\n",
       "Donners_2021             Medicine            0.817031\n",
       "Hall_2012                Computer science    0.694767\n",
       "Jeyaraman_2020           Medicine            0.906323\n",
       "Leenaars_2019            Medicine            0.243685\n",
       "                         Chemistry           0.245800\n",
       "                         Psychology          0.256428\n",
       "Leenaars_2020            Medicine            0.865833\n",
       "Meijboom_2021            Medicine            0.893434\n",
       "Menon_2022               Medicine            0.605102\n",
       "Moran_2021               Medicine            0.214867\n",
       "                         Biology             0.338177\n",
       "Muthu_2021               Medicine            0.950681\n",
       "Nelson_2002              Medicine            0.882701\n",
       "Oud_2018                 Medicine            0.262635\n",
       "                         Psychology          0.578095\n",
       "Radjenovic_2013          Computer science    0.715961\n",
       "Sep_2021                 Psychology          0.516370\n",
       "Smid_2020                Computer science    0.396219\n",
       "                         Mathematics         0.352785\n",
       "Walker_2018              Medicine            0.228265\n",
       "                         Biology             0.409869\n",
       "Wassenaar_2017           Medicine            0.318602\n",
       "                         Chemistry           0.202573\n",
       "                         Biology             0.310828\n",
       "Wolters_2018             Medicine            0.799883\n",
       "van_Dis_2020             Medicine            0.335423\n",
       "                         Psychology          0.500181\n",
       "van_de_Schoot_2018       Medicine            0.385027\n",
       "                         Psychology          0.439764\n",
       "van_der_Valk_2021        Medicine            0.553100\n",
       "                         Psychology          0.211837\n",
       "van_der_Waal_2022        Medicine            0.750499\n",
       "dtype: float64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "st_0n_incl = concepts_per_dataset_incl.stack()\n",
    "st_0n_incl[st_0n_incl >=0.2] \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f41315e0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>concept_publication</th>\n",
       "      <th>concept_search</th>\n",
       "      <th>concept_inclusions</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Appenzeller-Herzog_2019</th>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bos_2018</th>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Brouwer_2019</th>\n",
       "      <td>Psychology</td>\n",
       "      <td>Psychology</td>\n",
       "      <td>Psychology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Chou_2003</th>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Chou_2004</th>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Donners_2021</th>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hall_2012</th>\n",
       "      <td>Computer science</td>\n",
       "      <td>Computer science</td>\n",
       "      <td>Computer science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Jeyaraman_2020</th>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Leenaars_2019</th>\n",
       "      <td>Medicine</td>\n",
       "      <td>Chemistry</td>\n",
       "      <td>Psychology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Leenaars_2020</th>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Meijboom_2021</th>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Menon_2022</th>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Moran_2021</th>\n",
       "      <td>Psychology</td>\n",
       "      <td>Biology</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Muthu_2021</th>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Nelson_2002</th>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Oud_2018</th>\n",
       "      <td>Psychology</td>\n",
       "      <td>Psychology</td>\n",
       "      <td>Psychology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Radjenovic_2013</th>\n",
       "      <td>Computer science</td>\n",
       "      <td>Computer science</td>\n",
       "      <td>Computer science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sep_2021</th>\n",
       "      <td>Computer science</td>\n",
       "      <td>Psychology</td>\n",
       "      <td>Psychology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Smid_2020</th>\n",
       "      <td>Computer science</td>\n",
       "      <td>Mathematics</td>\n",
       "      <td>Computer science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>van_de_Schoot_2018</th>\n",
       "      <td>Computer science</td>\n",
       "      <td>Psychology</td>\n",
       "      <td>Psychology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>van_der_Valk_2021</th>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>van_der_Waal_2022</th>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>van_Dis_2020</th>\n",
       "      <td>Psychology</td>\n",
       "      <td>Psychology</td>\n",
       "      <td>Psychology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Walker_2018</th>\n",
       "      <td>Psychology</td>\n",
       "      <td>Medicine</td>\n",
       "      <td>Biology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wassenaar_2017</th>\n",
       "      <td>Medicine</td>\n",
       "      <td>Materials science</td>\n",
       "      <td>Medicine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wolters_2018</th>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "      <td>Medicine</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        concept_publication     concept_search   \n",
       "name                                                             \n",
       "Appenzeller-Herzog_2019            Medicine           Medicine  \\\n",
       "Bos_2018                           Medicine           Medicine   \n",
       "Brouwer_2019                     Psychology         Psychology   \n",
       "Chou_2003                          Medicine           Medicine   \n",
       "Chou_2004                          Medicine           Medicine   \n",
       "Donners_2021                       Medicine           Medicine   \n",
       "Hall_2012                  Computer science   Computer science   \n",
       "Jeyaraman_2020                     Medicine           Medicine   \n",
       "Leenaars_2019                      Medicine          Chemistry   \n",
       "Leenaars_2020                      Medicine           Medicine   \n",
       "Meijboom_2021                      Medicine           Medicine   \n",
       "Menon_2022                         Medicine           Medicine   \n",
       "Moran_2021                       Psychology            Biology   \n",
       "Muthu_2021                         Medicine           Medicine   \n",
       "Nelson_2002                        Medicine           Medicine   \n",
       "Oud_2018                         Psychology         Psychology   \n",
       "Radjenovic_2013            Computer science   Computer science   \n",
       "Sep_2021                   Computer science         Psychology   \n",
       "Smid_2020                  Computer science        Mathematics   \n",
       "van_de_Schoot_2018         Computer science         Psychology   \n",
       "van_der_Valk_2021                  Medicine           Medicine   \n",
       "van_der_Waal_2022                  Medicine           Medicine   \n",
       "van_Dis_2020                     Psychology         Psychology   \n",
       "Walker_2018                      Psychology           Medicine   \n",
       "Wassenaar_2017                     Medicine  Materials science   \n",
       "Wolters_2018                       Medicine           Medicine   \n",
       "\n",
       "                        concept_inclusions  \n",
       "name                                        \n",
       "Appenzeller-Herzog_2019           Medicine  \n",
       "Bos_2018                          Medicine  \n",
       "Brouwer_2019                    Psychology  \n",
       "Chou_2003                         Medicine  \n",
       "Chou_2004                         Medicine  \n",
       "Donners_2021                      Medicine  \n",
       "Hall_2012                 Computer science  \n",
       "Jeyaraman_2020                    Medicine  \n",
       "Leenaars_2019                   Psychology  \n",
       "Leenaars_2020                     Medicine  \n",
       "Meijboom_2021                     Medicine  \n",
       "Menon_2022                        Medicine  \n",
       "Moran_2021                         Biology  \n",
       "Muthu_2021                        Medicine  \n",
       "Nelson_2002                       Medicine  \n",
       "Oud_2018                        Psychology  \n",
       "Radjenovic_2013           Computer science  \n",
       "Sep_2021                        Psychology  \n",
       "Smid_2020                 Computer science  \n",
       "van_de_Schoot_2018              Psychology  \n",
       "van_der_Valk_2021                 Medicine  \n",
       "van_der_Waal_2022                 Medicine  \n",
       "van_Dis_2020                    Psychology  \n",
       "Walker_2018                        Biology  \n",
       "Wassenaar_2017                    Medicine  \n",
       "Wolters_2018                      Medicine  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pub.join(st[\"concept_search\"]).join(st_incl[\"concept_inclusions\"])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}