scispaCy_entitylinker.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "rlDmZCew2fI-"
      },
      "outputs": [],
      "source": [
        "%%capture\n",
        "!pip install scispacy\n",
        "!pip install 'https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.5.0/en_core_sci_md-0.5.0.tar.gz'"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import spacy\n",
        "import scispacy\n",
        "import pandas as pd\n",
        "import en_core_sci_md\n",
        "from scispacy.linking import EntityLinker"
      ],
      "metadata": {
        "id": "v5YY27Bl4Mo9"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "entities_dataframe = pd.read_csv('bionlp_entities.csv')"
      ],
      "metadata": {
        "id": "MG0hAXx_4RMb"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "entities_dataframe"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "XWltIkYWFqlX",
        "outputId": "33756642-13a3-4b68-86d2-94077fc23c03"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                       Entity                 Label   Ner_model\n",
              "0                    patients              ORGANISM  bionlp13cg\n",
              "1    loperamide hydrochloride       SIMPLE_CHEMICAL  bionlp13cg\n",
              "2             sodium chloride       SIMPLE_CHEMICAL  bionlp13cg\n",
              "3                   gut-liver    CELLULAR_COMPONENT  bionlp13cg\n",
              "4                 lymphocytes                  CELL  bionlp13cg\n",
              "..                        ...                   ...         ...\n",
              "96               electrolytes    CELLULAR_COMPONENT  bionlp13cg\n",
              "97                 lymphocyte                  CELL  bionlp13cg\n",
              "98         C-reactive protein  GENE_OR_GENE_PRODUCT  bionlp13cg\n",
              "99                glutathione       SIMPLE_CHEMICAL  bionlp13cg\n",
              "100                C-reactive  GENE_OR_GENE_PRODUCT  bionlp13cg\n",
              "\n",
              "[101 rows x 3 columns]"
            ],
            "text/html": [
              "\n",
              "\n",
              "  <div id=\"df-ec60400e-5b68-44b4-93a5-060232459aa1\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Entity</th>\n",
              "      <th>Label</th>\n",
              "      <th>Ner_model</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>patients</td>\n",
              "      <td>ORGANISM</td>\n",
              "      <td>bionlp13cg</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>loperamide hydrochloride</td>\n",
              "      <td>SIMPLE_CHEMICAL</td>\n",
              "      <td>bionlp13cg</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>sodium chloride</td>\n",
              "      <td>SIMPLE_CHEMICAL</td>\n",
              "      <td>bionlp13cg</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>gut-liver</td>\n",
              "      <td>CELLULAR_COMPONENT</td>\n",
              "      <td>bionlp13cg</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>lymphocytes</td>\n",
              "      <td>CELL</td>\n",
              "      <td>bionlp13cg</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>96</th>\n",
              "      <td>electrolytes</td>\n",
              "      <td>CELLULAR_COMPONENT</td>\n",
              "      <td>bionlp13cg</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>97</th>\n",
              "      <td>lymphocyte</td>\n",
              "      <td>CELL</td>\n",
              "      <td>bionlp13cg</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>98</th>\n",
              "      <td>C-reactive protein</td>\n",
              "      <td>GENE_OR_GENE_PRODUCT</td>\n",
              "      <td>bionlp13cg</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>99</th>\n",
              "      <td>glutathione</td>\n",
              "      <td>SIMPLE_CHEMICAL</td>\n",
              "      <td>bionlp13cg</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>100</th>\n",
              "      <td>C-reactive</td>\n",
              "      <td>GENE_OR_GENE_PRODUCT</td>\n",
              "      <td>bionlp13cg</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>101 rows × 3 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ec60400e-5b68-44b4-93a5-060232459aa1')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "\n",
              "\n",
              "\n",
              "    <div id=\"df-1c82c8e6-d99b-4f2c-818f-bdcd5a2fa038\">\n",
              "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-1c82c8e6-d99b-4f2c-818f-bdcd5a2fa038')\"\n",
              "              title=\"Suggest charts.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "      </button>\n",
              "    </div>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "    <script>\n",
              "      async function quickchart(key) {\n",
              "        const containerElement = document.querySelector('#' + key);\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      }\n",
              "    </script>\n",
              "\n",
              "      <script>\n",
              "\n",
              "function displayQuickchartButton(domScope) {\n",
              "  let quickchartButtonEl =\n",
              "    domScope.querySelector('#df-1c82c8e6-d99b-4f2c-818f-bdcd5a2fa038 button.colab-df-quickchart');\n",
              "  quickchartButtonEl.style.display =\n",
              "    google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "}\n",
              "\n",
              "        displayQuickchartButton(document);\n",
              "      </script>\n",
              "      <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-ec60400e-5b68-44b4-93a5-060232459aa1 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-ec60400e-5b68-44b4-93a5-060232459aa1');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "mesh_nlp = spacy.load(\"en_core_sci_md\")\n",
        "mesh_nlp.add_pipe(\"scispacy_linker\", config={\"resolve_abbreviations\": True, \"linker_name\": \"mesh\"}) #MeSH contains ~30k entities\n",
        "linker = mesh_nlp.get_pipe(\"scispacy_linker\")\n",
        "def mesh_entity_linker(document):\n",
        "    doc = mesh_nlp(document)\n",
        "    try:\n",
        "        entity = doc.ents[0]\n",
        "    except IndexError:\n",
        "        entity = 'Nan'\n",
        "    entity_details = []\n",
        "    entity_details.append(entity)\n",
        "    try:\n",
        "        for linker_ent in entity._.kb_ents:\n",
        "            Concept_Id, Score = linker_ent\n",
        "            entity_details.append('Entity_Matching_Score :{}'.format(Score))\n",
        "            entity_details.append(linker.kb.cui_to_entity[linker_ent[0]])\n",
        "    except AttributeError:\n",
        "        pass\n",
        "    return entity_details"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8JEkewHZFso-",
        "outputId": "ea0d559b-4711-4abb-e55b-94937d7e2dec"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "https://ai2-s2-scispacy.s3-us-west-2.amazonaws.com/data/linkers/2023-04-23/mesh/tfidf_vectors_sparse.npz not found in cache, downloading to /tmp/tmpjf69curo\n",
            "Finished download, copying /tmp/tmpjf69curo to cache at /root/.scispacy/datasets/0acb1f67e1908d2211efb5291880a946e905e1a14a87c10cfc640d0711f914c7.e4877c46bb5a882e9729b6abe799b33f195067557a3c0c15086a50471f29b985.tfidf_vectors_sparse.npz\n",
            "https://ai2-s2-scispacy.s3-us-west-2.amazonaws.com/data/linkers/2023-04-23/mesh/nmslib_index.bin not found in cache, downloading to /tmp/tmp49f0o902\n",
            "Finished download, copying /tmp/tmp49f0o902 to cache at /root/.scispacy/datasets/7bad4a37e60db48ee4b5b03dfaa61b195af5b4c69a6850fa5b466103229c263d.4952ca58f4ed53ad673bb387c8f203d92f422dbcc8cfb673ffed9720e7c0af68.nmslib_index.bin\n",
            "https://ai2-s2-scispacy.s3-us-west-2.amazonaws.com/data/linkers/2023-04-23/mesh/tfidf_vectorizer.joblib not found in cache, downloading to /tmp/tmpj0us0966\n",
            "Finished download, copying /tmp/tmpj0us0966 to cache at /root/.scispacy/datasets/6a0e66a77d89d942876d1b853abf461e6b16edaef65bebfa7f3a8dd99ff6553b.6eb17e1805a69a55fc151aa59fe42343d2b4be81405127043fd065bf5f9620e0.tfidf_vectorizer.joblib\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/sklearn/base.py:318: UserWarning: Trying to unpickle estimator TfidfTransformer from version 1.1.2 when using version 1.2.2. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\n",
            "https://scikit-learn.org/stable/model_persistence.html#security-maintainability-limitations\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/sklearn/base.py:318: UserWarning: Trying to unpickle estimator TfidfVectorizer from version 1.1.2 when using version 1.2.2. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\n",
            "https://scikit-learn.org/stable/model_persistence.html#security-maintainability-limitations\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "https://ai2-s2-scispacy.s3-us-west-2.amazonaws.com/data/linkers/2023-04-23/mesh/concept_aliases.json not found in cache, downloading to /tmp/tmpl0l_g4uy\n",
            "Finished download, copying /tmp/tmpl0l_g4uy to cache at /root/.scispacy/datasets/ccb3a55e3a37984902cc7de591d37d56d90eb0962d128536512b8d1219e71bcb.89e92a904a5ccc051bcba6ee26c5744e183dee7197cc835cfeb152b330b44559.concept_aliases.json\n",
            "https://ai2-s2-scispacy.s3-us-west-2.amazonaws.com/data/kbs/2023-04-23/umls_mesh_2022.jsonl not found in cache, downloading to /tmp/tmp3uwj1w9_\n",
            "Finished download, copying /tmp/tmp3uwj1w9_ to cache at /root/.scispacy/datasets/5541a1df25533cfafec1fdcf0446c761f998591519c8ad4a73876f48d7e0a224.c4d6e393746f18aaf6eafff94fe1782cebf29ef535b101501e66f1e3462cdb09.umls_mesh_2022.jsonl\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "entities_dataframe['mesh_output'] = entities_dataframe['Entity'].apply(lambda x : mesh_entity_linker(x))"
      ],
      "metadata": {
        "id": "K-zzALfcGqSf"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "entities_dataframe"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "PGi3tkQIGu4b",
        "outputId": "a6a44570-5a24-4255-a133-6b9abc1c8821"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                       Entity                 Label   Ner_model  \\\n",
              "0                    patients              ORGANISM  bionlp13cg   \n",
              "1    loperamide hydrochloride       SIMPLE_CHEMICAL  bionlp13cg   \n",
              "2             sodium chloride       SIMPLE_CHEMICAL  bionlp13cg   \n",
              "3                   gut-liver    CELLULAR_COMPONENT  bionlp13cg   \n",
              "4                 lymphocytes                  CELL  bionlp13cg   \n",
              "..                        ...                   ...         ...   \n",
              "96               electrolytes    CELLULAR_COMPONENT  bionlp13cg   \n",
              "97                 lymphocyte                  CELL  bionlp13cg   \n",
              "98         C-reactive protein  GENE_OR_GENE_PRODUCT  bionlp13cg   \n",
              "99                glutathione       SIMPLE_CHEMICAL  bionlp13cg   \n",
              "100                C-reactive  GENE_OR_GENE_PRODUCT  bionlp13cg   \n",
              "\n",
              "                                           mesh_output  \n",
              "0    [(patients), Entity_Matching_Score :1.0, (C003...  \n",
              "1    [(loperamide, hydrochloride), Entity_Matching_...  \n",
              "2    [(sodium, chloride), Entity_Matching_Score :1....  \n",
              "3                                        [(gut-liver)]  \n",
              "4    [(lymphocytes), Entity_Matching_Score :1.0, (C...  \n",
              "..                                                 ...  \n",
              "96   [(electrolytes), Entity_Matching_Score :1.0, (...  \n",
              "97   [(lymphocyte), Entity_Matching_Score :1.0, (C0...  \n",
              "98   [(C-reactive, protein), Entity_Matching_Score ...  \n",
              "99   [(glutathione), Entity_Matching_Score :1.0, (C...  \n",
              "100                                              [Nan]  \n",
              "\n",
              "[101 rows x 4 columns]"
            ],
            "text/html": [
              "\n",
              "\n",
              "  <div id=\"df-6b812e07-5289-4031-9e0f-99bf40183b84\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Entity</th>\n",
              "      <th>Label</th>\n",
              "      <th>Ner_model</th>\n",
              "      <th>mesh_output</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>patients</td>\n",
              "      <td>ORGANISM</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(patients), Entity_Matching_Score :1.0, (C003...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>loperamide hydrochloride</td>\n",
              "      <td>SIMPLE_CHEMICAL</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(loperamide, hydrochloride), Entity_Matching_...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>sodium chloride</td>\n",
              "      <td>SIMPLE_CHEMICAL</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(sodium, chloride), Entity_Matching_Score :1....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>gut-liver</td>\n",
              "      <td>CELLULAR_COMPONENT</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(gut-liver)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>lymphocytes</td>\n",
              "      <td>CELL</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(lymphocytes), Entity_Matching_Score :1.0, (C...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>96</th>\n",
              "      <td>electrolytes</td>\n",
              "      <td>CELLULAR_COMPONENT</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(electrolytes), Entity_Matching_Score :1.0, (...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>97</th>\n",
              "      <td>lymphocyte</td>\n",
              "      <td>CELL</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(lymphocyte), Entity_Matching_Score :1.0, (C0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>98</th>\n",
              "      <td>C-reactive protein</td>\n",
              "      <td>GENE_OR_GENE_PRODUCT</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(C-reactive, protein), Entity_Matching_Score ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>99</th>\n",
              "      <td>glutathione</td>\n",
              "      <td>SIMPLE_CHEMICAL</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(glutathione), Entity_Matching_Score :1.0, (C...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>100</th>\n",
              "      <td>C-reactive</td>\n",
              "      <td>GENE_OR_GENE_PRODUCT</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[Nan]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>101 rows × 4 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6b812e07-5289-4031-9e0f-99bf40183b84')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "\n",
              "\n",
              "\n",
              "    <div id=\"df-f397f052-06fa-4714-ba8a-4030ae0e692d\">\n",
              "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-f397f052-06fa-4714-ba8a-4030ae0e692d')\"\n",
              "              title=\"Suggest charts.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "      </button>\n",
              "    </div>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "    <script>\n",
              "      async function quickchart(key) {\n",
              "        const containerElement = document.querySelector('#' + key);\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      }\n",
              "    </script>\n",
              "\n",
              "      <script>\n",
              "\n",
              "function displayQuickchartButton(domScope) {\n",
              "  let quickchartButtonEl =\n",
              "    domScope.querySelector('#df-f397f052-06fa-4714-ba8a-4030ae0e692d button.colab-df-quickchart');\n",
              "  quickchartButtonEl.style.display =\n",
              "    google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "}\n",
              "\n",
              "        displayQuickchartButton(document);\n",
              "      </script>\n",
              "      <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6b812e07-5289-4031-9e0f-99bf40183b84 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6b812e07-5289-4031-9e0f-99bf40183b84');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "entities_dataframe['mesh_output'][0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-7oQ4as5GyPU",
        "outputId": "d299eb9a-54a8-437d-c6e3-154b4d9daeb9"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[patients,\n",
              " 'Entity_Matching_Score :1.0',\n",
              " CUI: C0030705, Name: Patients\n",
              " Definition: Individuals participating in the health care system for the purpose of receiving therapeutic, diagnostic, or preventive procedures.\n",
              " TUI(s): T101\n",
              " Aliases: (total: 1): \n",
              " \t Patient,\n",
              " 'Entity_Matching_Score :0.8274745941162109',\n",
              " CUI: C0025360, Name: Mentally Ill Persons\n",
              " Definition: Persons with psychiatric illnesses or diseases, particularly psychotic and severe mood disorders.\n",
              " TUI(s): T101\n",
              " Aliases: (total: 6): \n",
              " \t Mental Patients, Person, Mentally Ill, Ill, Mentally, Mentally Ill Person, Persons, Mentally Ill, Mentally Ill,\n",
              " 'Entity_Matching_Score :0.7562460899353027',\n",
              " CUI: C0029921, Name: Outpatients\n",
              " Definition: Persons who receive ambulatory care at an outpatient department or clinic without room and board being provided.\n",
              " TUI(s): T101\n",
              " Aliases: (total: 4): \n",
              " \t Out-patients, Out-patient, Out patients, Outpatient,\n",
              " 'Entity_Matching_Score :0.7128358483314514',\n",
              " CUI: C0030706, Name: Right, Patients'\n",
              " Definition: Fundamental claims of patients, as expressed in statutes, declarations, or generally accepted moral principles. (Bioethics Thesaurus) The term is used for discussions of patient rights as a group of many rights, as in a hospital's posting of a list of patient rights.\n",
              " TUI(s): T078\n",
              " Aliases (abbreviated, total: 12): \n",
              " \t Patients' Rights, Patient's Rights, Patients' Right, Patient's Right, Rights, Patient, Right, Patient, Rights, Patient's, Patient Right, Patients Rights, Patient Rights,\n",
              " 'Entity_Matching_Score :0.711505115032196',\n",
              " CUI: C0042801, Name: Visitors to Patients\n",
              " Definition: Patients' guests and rules for visiting.\n",
              " TUI(s): T098\n",
              " Aliases: (total: 8): \n",
              " \t Patients Visitors, Patients' Visitors, Patient Visitors, Patient's Visitors, Patients' Visitor, Visitor, Patients', Visitors to Patient, Visitors, Patients']"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "hpo_nlp = spacy.load(\"en_core_sci_md\")\n",
        "hpo_nlp.add_pipe(\"scispacy_linker\", config={\"resolve_abbreviations\": True, \"linker_name\": \"hpo\"}) #16k concepts focused on phenotypic abnormalities encountered in human disease.\n",
        "linker = hpo_nlp.get_pipe(\"scispacy_linker\")\n",
        "def hpo_entity_linker(document):\n",
        "    doc = hpo_nlp(document)\n",
        "    try:\n",
        "        entity = doc.ents[0]\n",
        "    except IndexError:\n",
        "        entity = 'Nan'\n",
        "    entity_details = []\n",
        "    entity_details.append(entity)\n",
        "    try:\n",
        "        for linker_ent in entity._.kb_ents:\n",
        "            Concept_Id, Score = linker_ent\n",
        "            entity_details.append('Entity_Matching_Score :{}'.format(Score))\n",
        "            entity_details.append(linker.kb.cui_to_entity[linker_ent[0]])\n",
        "    except AttributeError:\n",
        "        pass\n",
        "    return entity_details"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VrYTcPK0G4zH",
        "outputId": "e722bdf2-11ec-4f86-a98c-b6d20ae9434c"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "https://ai2-s2-scispacy.s3-us-west-2.amazonaws.com/data/linkers/2023-04-23/hpo/tfidf_vectors_sparse.npz not found in cache, downloading to /tmp/tmpym8njx3e\n",
            "Finished download, copying /tmp/tmpym8njx3e to cache at /root/.scispacy/datasets/ce11d8a176fa1830308fc265ab8845ca877f10c70fa3f74212ff2d9fdd97ab96.029e8ca566e1b5d6ab99138a96aa1c7b050565132aabb6b296a1c870c64d6f9b.tfidf_vectors_sparse.npz\n",
            "https://ai2-s2-scispacy.s3-us-west-2.amazonaws.com/data/linkers/2023-04-23/hpo/nmslib_index.bin not found in cache, downloading to /tmp/tmp8aa91bf8\n",
            "Finished download, copying /tmp/tmp8aa91bf8 to cache at /root/.scispacy/datasets/066d3db776b9acaff67728a857a1d6625f4c86194a70804ffd5399fa738caa4e.ecc1ac28794235140b2bafbbf81ce0454219cd1e05056786dce65ab17fee53b2.nmslib_index.bin\n",
            "https://ai2-s2-scispacy.s3-us-west-2.amazonaws.com/data/linkers/2023-04-23/hpo/tfidf_vectorizer.joblib not found in cache, downloading to /tmp/tmpx1vgcmlo\n",
            "Finished download, copying /tmp/tmpx1vgcmlo to cache at /root/.scispacy/datasets/cdf67b07073317b3e1b0773eee16408261d3748e12bacb00a05eedf8de0daca5.8b3ec883bcab0da944125277caaacbcf72880dba390effff464cda4d2dcded62.tfidf_vectorizer.joblib\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/sklearn/base.py:318: UserWarning: Trying to unpickle estimator TfidfTransformer from version 1.1.2 when using version 1.2.2. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\n",
            "https://scikit-learn.org/stable/model_persistence.html#security-maintainability-limitations\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/sklearn/base.py:318: UserWarning: Trying to unpickle estimator TfidfVectorizer from version 1.1.2 when using version 1.2.2. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\n",
            "https://scikit-learn.org/stable/model_persistence.html#security-maintainability-limitations\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "https://ai2-s2-scispacy.s3-us-west-2.amazonaws.com/data/linkers/2023-04-23/hpo/concept_aliases.json not found in cache, downloading to /tmp/tmpcqy_ty7p\n",
            "Finished download, copying /tmp/tmpcqy_ty7p to cache at /root/.scispacy/datasets/092c266817935c16682d3a1511bad5bdb7e3665d93da4d2eb21d42fa6b2f4100.298fa9e3ef85c61367c35b4240deae6f06545e2cb68659bbad65602be2dfefab.concept_aliases.json\n",
            "https://ai2-s2-scispacy.s3-us-west-2.amazonaws.com/data/kbs/2023-04-23/umls_hpo_2022.jsonl not found in cache, downloading to /tmp/tmp6wpuhnp3\n",
            "Finished download, copying /tmp/tmp6wpuhnp3 to cache at /root/.scispacy/datasets/4acfb77195a577a57a9791f9627dcc8c47561d8c2fa4671b9a5ca0e494970e87.b703e72c55ea536eac9c2fcb2d63553d36ea0aadec6a3f525a9eb21998302bc7.umls_hpo_2022.jsonl\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "entities_dataframe['hpo_output'] = entities_dataframe['Entity'].apply(lambda x : hpo_entity_linker(x))"
      ],
      "metadata": {
        "id": "z8pF4UhsG7EV"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "entities_dataframe"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "id": "up5dyVVZHBOt",
        "outputId": "863fcb1f-0efc-493b-d3fa-3d2893327edd"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                       Entity                 Label   Ner_model  \\\n",
              "0                    patients              ORGANISM  bionlp13cg   \n",
              "1    loperamide hydrochloride       SIMPLE_CHEMICAL  bionlp13cg   \n",
              "2             sodium chloride       SIMPLE_CHEMICAL  bionlp13cg   \n",
              "3                   gut-liver    CELLULAR_COMPONENT  bionlp13cg   \n",
              "4                 lymphocytes                  CELL  bionlp13cg   \n",
              "..                        ...                   ...         ...   \n",
              "96               electrolytes    CELLULAR_COMPONENT  bionlp13cg   \n",
              "97                 lymphocyte                  CELL  bionlp13cg   \n",
              "98         C-reactive protein  GENE_OR_GENE_PRODUCT  bionlp13cg   \n",
              "99                glutathione       SIMPLE_CHEMICAL  bionlp13cg   \n",
              "100                C-reactive  GENE_OR_GENE_PRODUCT  bionlp13cg   \n",
              "\n",
              "                                           mesh_output  \\\n",
              "0    [(patients), Entity_Matching_Score :1.0, (C003...   \n",
              "1    [(loperamide, hydrochloride), Entity_Matching_...   \n",
              "2    [(sodium, chloride), Entity_Matching_Score :1....   \n",
              "3                                        [(gut-liver)]   \n",
              "4    [(lymphocytes), Entity_Matching_Score :1.0, (C...   \n",
              "..                                                 ...   \n",
              "96   [(electrolytes), Entity_Matching_Score :1.0, (...   \n",
              "97   [(lymphocyte), Entity_Matching_Score :1.0, (C0...   \n",
              "98   [(C-reactive, protein), Entity_Matching_Score ...   \n",
              "99   [(glutathione), Entity_Matching_Score :1.0, (C...   \n",
              "100                                              [Nan]   \n",
              "\n",
              "                                            hpo_output  \n",
              "0                                         [(patients)]  \n",
              "1                        [(loperamide, hydrochloride)]  \n",
              "2                                 [(sodium, chloride)]  \n",
              "3                                        [(gut-liver)]  \n",
              "4    [(lymphocytes), Entity_Matching_Score :0.90991...  \n",
              "..                                                 ...  \n",
              "96   [(electrolytes), Entity_Matching_Score :0.7037...  \n",
              "97   [(lymphocyte), Entity_Matching_Score :0.831573...  \n",
              "98   [(C-reactive, protein), Entity_Matching_Score ...  \n",
              "99                                     [(glutathione)]  \n",
              "100                                              [Nan]  \n",
              "\n",
              "[101 rows x 5 columns]"
            ],
            "text/html": [
              "\n",
              "\n",
              "  <div id=\"df-bc30bd6b-abe5-45d2-95ac-85377321e90e\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Entity</th>\n",
              "      <th>Label</th>\n",
              "      <th>Ner_model</th>\n",
              "      <th>mesh_output</th>\n",
              "      <th>hpo_output</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>patients</td>\n",
              "      <td>ORGANISM</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(patients), Entity_Matching_Score :1.0, (C003...</td>\n",
              "      <td>[(patients)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>loperamide hydrochloride</td>\n",
              "      <td>SIMPLE_CHEMICAL</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(loperamide, hydrochloride), Entity_Matching_...</td>\n",
              "      <td>[(loperamide, hydrochloride)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>sodium chloride</td>\n",
              "      <td>SIMPLE_CHEMICAL</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(sodium, chloride), Entity_Matching_Score :1....</td>\n",
              "      <td>[(sodium, chloride)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>gut-liver</td>\n",
              "      <td>CELLULAR_COMPONENT</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(gut-liver)]</td>\n",
              "      <td>[(gut-liver)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>lymphocytes</td>\n",
              "      <td>CELL</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(lymphocytes), Entity_Matching_Score :1.0, (C...</td>\n",
              "      <td>[(lymphocytes), Entity_Matching_Score :0.90991...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>96</th>\n",
              "      <td>electrolytes</td>\n",
              "      <td>CELLULAR_COMPONENT</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(electrolytes), Entity_Matching_Score :1.0, (...</td>\n",
              "      <td>[(electrolytes), Entity_Matching_Score :0.7037...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>97</th>\n",
              "      <td>lymphocyte</td>\n",
              "      <td>CELL</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(lymphocyte), Entity_Matching_Score :1.0, (C0...</td>\n",
              "      <td>[(lymphocyte), Entity_Matching_Score :0.831573...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>98</th>\n",
              "      <td>C-reactive protein</td>\n",
              "      <td>GENE_OR_GENE_PRODUCT</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(C-reactive, protein), Entity_Matching_Score ...</td>\n",
              "      <td>[(C-reactive, protein), Entity_Matching_Score ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>99</th>\n",
              "      <td>glutathione</td>\n",
              "      <td>SIMPLE_CHEMICAL</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(glutathione), Entity_Matching_Score :1.0, (C...</td>\n",
              "      <td>[(glutathione)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>100</th>\n",
              "      <td>C-reactive</td>\n",
              "      <td>GENE_OR_GENE_PRODUCT</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[Nan]</td>\n",
              "      <td>[Nan]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>101 rows × 5 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-bc30bd6b-abe5-45d2-95ac-85377321e90e')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "\n",
              "\n",
              "\n",
              "    <div id=\"df-ed470826-6648-4103-a5c0-352eaacc6fae\">\n",
              "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-ed470826-6648-4103-a5c0-352eaacc6fae')\"\n",
              "              title=\"Suggest charts.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "      </button>\n",
              "    </div>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "    <script>\n",
              "      async function quickchart(key) {\n",
              "        const containerElement = document.querySelector('#' + key);\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      }\n",
              "    </script>\n",
              "\n",
              "      <script>\n",
              "\n",
              "function displayQuickchartButton(domScope) {\n",
              "  let quickchartButtonEl =\n",
              "    domScope.querySelector('#df-ed470826-6648-4103-a5c0-352eaacc6fae button.colab-df-quickchart');\n",
              "  quickchartButtonEl.style.display =\n",
              "    google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "}\n",
              "\n",
              "        displayQuickchartButton(document);\n",
              "      </script>\n",
              "      <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-bc30bd6b-abe5-45d2-95ac-85377321e90e button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-bc30bd6b-abe5-45d2-95ac-85377321e90e');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "entities_dataframe['hpo_output'][4]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q4xMLvJSHEmp",
        "outputId": "4ba09758-9498-4f2b-8e09-3e7841c616f6"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[lymphocytes,\n",
              " 'Entity_Matching_Score :0.9099137783050537',\n",
              " CUI: C0221277, Name: Abnormal lymphocyte morphology\n",
              " Definition: A lymphocyte that may be irregular or not conforming to type.\n",
              " TUI(s): T033\n",
              " Aliases: (total: 2): \n",
              " \t Abnormal lymphocytes, Abnormality of cells of the lymphoid lineage,\n",
              " 'Entity_Matching_Score :0.7441422939300537',\n",
              " CUI: C0580550, Name: Abnormal numbers of lymphocytes\n",
              " Definition: Any abnormality in the total number of lymphocytes in the blood. []\n",
              " TUI(s): T033\n",
              " Aliases: (total: 4): \n",
              " \t Abnormal number of lymphocytes, Abnormality of lymphocyte number, Abnormal lymphocyte count, Abnormal lymphocyte counts,\n",
              " 'Entity_Matching_Score :0.7385052442550659',\n",
              " CUI: C0024282, Name: High lymphocyte count\n",
              " Definition: Excess of normal lymphocytes in the blood or in any effusion.\n",
              " TUI(s): T047\n",
              " Aliases: (total: 1): \n",
              " \t Lymphocytosis]"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "rxnorm_nlp = spacy.load(\"en_core_sci_md\")\n",
        "rxnorm_nlp.add_pipe(\"scispacy_linker\", config={\"resolve_abbreviations\": True, \"linker_name\": \"rxnorm\"}) #RxNorm contains ~100k concepts focused on normalized names for clinical drugs\n",
        "linker = rxnorm_nlp.get_pipe(\"scispacy_linker\")\n",
        "def rxnorm_entity_linker(document):\n",
        "    doc = rxnorm_nlp(document)\n",
        "    try:\n",
        "        entity = doc.ents[0]\n",
        "    except IndexError:\n",
        "        entity = 'Nan'\n",
        "    entity_details = []\n",
        "    entity_details.append(entity)\n",
        "    try:\n",
        "        for linker_ent in entity._.kb_ents:\n",
        "            Concept_Id, Score = linker_ent\n",
        "            entity_details.append('Entity_Matching_Score :{}'.format(Score))\n",
        "            entity_details.append(linker.kb.cui_to_entity[linker_ent[0]])\n",
        "    except AttributeError:\n",
        "        pass\n",
        "    return entity_details"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tNCNgC7uHIue",
        "outputId": "db495aa2-fc2b-4f77-fd6b-8efc53e8c65b"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "https://ai2-s2-scispacy.s3-us-west-2.amazonaws.com/data/linkers/2023-04-23/rxnorm/tfidf_vectors_sparse.npz not found in cache, downloading to /tmp/tmpbfoeg88x\n",
            "Finished download, copying /tmp/tmpbfoeg88x to cache at /root/.scispacy/datasets/68e7f1197d5852698808a5f9d694026c210e4b93a7e496dea608a46fff914774.e9a1075d5c32b5e7a180b60a96b15fc072ea714b95dd458047a48ccf2bb065be.tfidf_vectors_sparse.npz\n",
            "https://ai2-s2-scispacy.s3-us-west-2.amazonaws.com/data/linkers/2023-04-23/rxnorm/nmslib_index.bin not found in cache, downloading to /tmp/tmpismu69yo\n",
            "Finished download, copying /tmp/tmpismu69yo to cache at /root/.scispacy/datasets/3742ff1d61c637ce7dc935674fa4199810af16978f9a10088d71d37bba16203a.8f798c6f751125a0d68f8b4e82ecfba4fd37bfb2a447d61fba584e208e6af9d3.nmslib_index.bin\n",
            "https://ai2-s2-scispacy.s3-us-west-2.amazonaws.com/data/linkers/2023-04-23/rxnorm/tfidf_vectorizer.joblib not found in cache, downloading to /tmp/tmp155d06dj\n",
            "Finished download, copying /tmp/tmp155d06dj to cache at /root/.scispacy/datasets/e6db3b626658739bfbd89a4695141db556c21cb8b915a8e7de00650992529158.2bf384392e4cece70fca03154737daf5a4e8a43fcab3fe83bb8e5d3467ccaff1.tfidf_vectorizer.joblib\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/sklearn/base.py:318: UserWarning: Trying to unpickle estimator TfidfTransformer from version 1.1.2 when using version 1.2.2. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\n",
            "https://scikit-learn.org/stable/model_persistence.html#security-maintainability-limitations\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/sklearn/base.py:318: UserWarning: Trying to unpickle estimator TfidfVectorizer from version 1.1.2 when using version 1.2.2. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\n",
            "https://scikit-learn.org/stable/model_persistence.html#security-maintainability-limitations\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "https://ai2-s2-scispacy.s3-us-west-2.amazonaws.com/data/linkers/2023-04-23/rxnorm/concept_aliases.json not found in cache, downloading to /tmp/tmpdvh0hyoc\n",
            "Finished download, copying /tmp/tmpdvh0hyoc to cache at /root/.scispacy/datasets/54a3afac2f157748a3326a13e59ffe165fcc40ce0cceab6dc47303965dc3c0ed.71746c536649e7ba8a47b6cb7a3a7c8e0c447e022bdf819e69fbb1de9276d411.concept_aliases.json\n",
            "https://ai2-s2-scispacy.s3-us-west-2.amazonaws.com/data/kbs/2023-04-23/umls_rxnorm_2022.jsonl not found in cache, downloading to /tmp/tmp0x2fhn5v\n",
            "Finished download, copying /tmp/tmp0x2fhn5v to cache at /root/.scispacy/datasets/afd8034c6b1a9b6e9eb94a5688ab043023fb450ddf36c88b9f78efa21c5b2d0a.7afae38a116c40277e6052ddcfcd0013fb8136a6d4f96d965ccc7689e8543712.umls_rxnorm_2022.jsonl\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "entities_dataframe['rxnorm_output'] = entities_dataframe['Entity'].apply(lambda x : rxnorm_entity_linker(x))"
      ],
      "metadata": {
        "id": "vDVD_Vu0HNUV"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "entities_dataframe"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 467
        },
        "id": "evjQmvaHHOQS",
        "outputId": "1b7a84d3-f412-4ee7-accf-af1f0d739edf"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                       Entity                 Label   Ner_model  \\\n",
              "0                    patients              ORGANISM  bionlp13cg   \n",
              "1    loperamide hydrochloride       SIMPLE_CHEMICAL  bionlp13cg   \n",
              "2             sodium chloride       SIMPLE_CHEMICAL  bionlp13cg   \n",
              "3                   gut-liver    CELLULAR_COMPONENT  bionlp13cg   \n",
              "4                 lymphocytes                  CELL  bionlp13cg   \n",
              "..                        ...                   ...         ...   \n",
              "96               electrolytes    CELLULAR_COMPONENT  bionlp13cg   \n",
              "97                 lymphocyte                  CELL  bionlp13cg   \n",
              "98         C-reactive protein  GENE_OR_GENE_PRODUCT  bionlp13cg   \n",
              "99                glutathione       SIMPLE_CHEMICAL  bionlp13cg   \n",
              "100                C-reactive  GENE_OR_GENE_PRODUCT  bionlp13cg   \n",
              "\n",
              "                                           mesh_output  \\\n",
              "0    [(patients), Entity_Matching_Score :1.0, (C003...   \n",
              "1    [(loperamide, hydrochloride), Entity_Matching_...   \n",
              "2    [(sodium, chloride), Entity_Matching_Score :1....   \n",
              "3                                        [(gut-liver)]   \n",
              "4    [(lymphocytes), Entity_Matching_Score :1.0, (C...   \n",
              "..                                                 ...   \n",
              "96   [(electrolytes), Entity_Matching_Score :1.0, (...   \n",
              "97   [(lymphocyte), Entity_Matching_Score :1.0, (C0...   \n",
              "98   [(C-reactive, protein), Entity_Matching_Score ...   \n",
              "99   [(glutathione), Entity_Matching_Score :1.0, (C...   \n",
              "100                                              [Nan]   \n",
              "\n",
              "                                            hpo_output  \\\n",
              "0                                         [(patients)]   \n",
              "1                        [(loperamide, hydrochloride)]   \n",
              "2                                 [(sodium, chloride)]   \n",
              "3                                        [(gut-liver)]   \n",
              "4    [(lymphocytes), Entity_Matching_Score :0.90991...   \n",
              "..                                                 ...   \n",
              "96   [(electrolytes), Entity_Matching_Score :0.7037...   \n",
              "97   [(lymphocyte), Entity_Matching_Score :0.831573...   \n",
              "98   [(C-reactive, protein), Entity_Matching_Score ...   \n",
              "99                                     [(glutathione)]   \n",
              "100                                              [Nan]   \n",
              "\n",
              "                                         rxnorm_output  \n",
              "0                                         [(patients)]  \n",
              "1    [(loperamide, hydrochloride), Entity_Matching_...  \n",
              "2    [(sodium, chloride), Entity_Matching_Score :1....  \n",
              "3                                        [(gut-liver)]  \n",
              "4                                      [(lymphocytes)]  \n",
              "..                                                 ...  \n",
              "96                                    [(electrolytes)]  \n",
              "97                                      [(lymphocyte)]  \n",
              "98                             [(C-reactive, protein)]  \n",
              "99   [(glutathione), Entity_Matching_Score :1.0, (C...  \n",
              "100                                              [Nan]  \n",
              "\n",
              "[101 rows x 6 columns]"
            ],
            "text/html": [
              "\n",
              "\n",
              "  <div id=\"df-3675581c-d186-4f5e-a82f-30a6281a13be\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Entity</th>\n",
              "      <th>Label</th>\n",
              "      <th>Ner_model</th>\n",
              "      <th>mesh_output</th>\n",
              "      <th>hpo_output</th>\n",
              "      <th>rxnorm_output</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>patients</td>\n",
              "      <td>ORGANISM</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(patients), Entity_Matching_Score :1.0, (C003...</td>\n",
              "      <td>[(patients)]</td>\n",
              "      <td>[(patients)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>loperamide hydrochloride</td>\n",
              "      <td>SIMPLE_CHEMICAL</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(loperamide, hydrochloride), Entity_Matching_...</td>\n",
              "      <td>[(loperamide, hydrochloride)]</td>\n",
              "      <td>[(loperamide, hydrochloride), Entity_Matching_...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>sodium chloride</td>\n",
              "      <td>SIMPLE_CHEMICAL</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(sodium, chloride), Entity_Matching_Score :1....</td>\n",
              "      <td>[(sodium, chloride)]</td>\n",
              "      <td>[(sodium, chloride), Entity_Matching_Score :1....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>gut-liver</td>\n",
              "      <td>CELLULAR_COMPONENT</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(gut-liver)]</td>\n",
              "      <td>[(gut-liver)]</td>\n",
              "      <td>[(gut-liver)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>lymphocytes</td>\n",
              "      <td>CELL</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(lymphocytes), Entity_Matching_Score :1.0, (C...</td>\n",
              "      <td>[(lymphocytes), Entity_Matching_Score :0.90991...</td>\n",
              "      <td>[(lymphocytes)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>96</th>\n",
              "      <td>electrolytes</td>\n",
              "      <td>CELLULAR_COMPONENT</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(electrolytes), Entity_Matching_Score :1.0, (...</td>\n",
              "      <td>[(electrolytes), Entity_Matching_Score :0.7037...</td>\n",
              "      <td>[(electrolytes)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>97</th>\n",
              "      <td>lymphocyte</td>\n",
              "      <td>CELL</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(lymphocyte), Entity_Matching_Score :1.0, (C0...</td>\n",
              "      <td>[(lymphocyte), Entity_Matching_Score :0.831573...</td>\n",
              "      <td>[(lymphocyte)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>98</th>\n",
              "      <td>C-reactive protein</td>\n",
              "      <td>GENE_OR_GENE_PRODUCT</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(C-reactive, protein), Entity_Matching_Score ...</td>\n",
              "      <td>[(C-reactive, protein), Entity_Matching_Score ...</td>\n",
              "      <td>[(C-reactive, protein)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>99</th>\n",
              "      <td>glutathione</td>\n",
              "      <td>SIMPLE_CHEMICAL</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(glutathione), Entity_Matching_Score :1.0, (C...</td>\n",
              "      <td>[(glutathione)]</td>\n",
              "      <td>[(glutathione), Entity_Matching_Score :1.0, (C...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>100</th>\n",
              "      <td>C-reactive</td>\n",
              "      <td>GENE_OR_GENE_PRODUCT</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[Nan]</td>\n",
              "      <td>[Nan]</td>\n",
              "      <td>[Nan]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>101 rows × 6 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3675581c-d186-4f5e-a82f-30a6281a13be')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "\n",
              "\n",
              "\n",
              "    <div id=\"df-43df5a33-915f-4457-9f20-c4dfb5e615cf\">\n",
              "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-43df5a33-915f-4457-9f20-c4dfb5e615cf')\"\n",
              "              title=\"Suggest charts.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "      </button>\n",
              "    </div>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "    <script>\n",
              "      async function quickchart(key) {\n",
              "        const containerElement = document.querySelector('#' + key);\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      }\n",
              "    </script>\n",
              "\n",
              "      <script>\n",
              "\n",
              "function displayQuickchartButton(domScope) {\n",
              "  let quickchartButtonEl =\n",
              "    domScope.querySelector('#df-43df5a33-915f-4457-9f20-c4dfb5e615cf button.colab-df-quickchart');\n",
              "  quickchartButtonEl.style.display =\n",
              "    google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "}\n",
              "\n",
              "        displayQuickchartButton(document);\n",
              "      </script>\n",
              "      <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-3675581c-d186-4f5e-a82f-30a6281a13be button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-3675581c-d186-4f5e-a82f-30a6281a13be');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "entities_dataframe['rxnorm_output'][99]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qa1ceYufHhN-",
        "outputId": "79c2b7bd-f694-40e7-a834-a4a9a0def5f7"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[glutathione,\n",
              " 'Entity_Matching_Score :1.0',\n",
              " CUI: C0017817, Name: glutathione\n",
              " Definition: A tripeptide with many roles in cells. It conjugates to drugs to make them more soluble for excretion, is a cofactor for some enzymes, is involved in protein disulfide bond rearrangement and reduces peroxides.\n",
              " TUI(s): T116, T121, T123\n",
              " Aliases: (total: 1): \n",
              " \t L-glutathione]"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "go_nlp = spacy.load(\"en_core_sci_md\")\n",
        "go_nlp.add_pipe(\"scispacy_linker\", config={\"resolve_abbreviations\": True, \"linker_name\": \"go\"}) #Gene Ontology contains ~67k concepts focused on the functions of genes\n",
        "linker = go_nlp.get_pipe(\"scispacy_linker\")\n",
        "def go_entity_linker(document):\n",
        "    doc = go_nlp(document)\n",
        "    try:\n",
        "        entity = doc.ents[0]\n",
        "    except IndexError:\n",
        "        entity = 'Nan'\n",
        "    entity_details = []\n",
        "    entity_details.append(entity)\n",
        "    try:\n",
        "        for linker_ent in entity._.kb_ents:\n",
        "            Concept_Id, Score = linker_ent\n",
        "            entity_details.append('Entity_Matching_Score :{}'.format(Score))\n",
        "            entity_details.append(linker.kb.cui_to_entity[linker_ent[0]])\n",
        "    except AttributeError:\n",
        "        pass\n",
        "    return entity_details"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nSJsItbgHjrt",
        "outputId": "52361d4e-f1d3-4101-ecda-b8d780320e64"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "https://ai2-s2-scispacy.s3-us-west-2.amazonaws.com/data/linkers/2023-04-23/go/tfidf_vectors_sparse.npz not found in cache, downloading to /tmp/tmpwjhowrey\n",
            "Finished download, copying /tmp/tmpwjhowrey to cache at /root/.scispacy/datasets/98b21d1968addfd51eceee816a491b7b10de52fbc8f11f22fbf8374d9f881229.0a8a2035151feef72cf9dc0bcda27bda35e86771810a2a4523bae7ea337ae7bb.tfidf_vectors_sparse.npz\n",
            "https://ai2-s2-scispacy.s3-us-west-2.amazonaws.com/data/linkers/2023-04-23/go/nmslib_index.bin not found in cache, downloading to /tmp/tmpc_bap78w\n",
            "Finished download, copying /tmp/tmpc_bap78w to cache at /root/.scispacy/datasets/3ed448934f89223c37be21a402a665d6e3dfcbea9bfd87b1fcd68dbb2f850760.40c7e42a18bea0b2f632b9ec6c299545f1f7d91b2187158ee03380d639eb867f.nmslib_index.bin\n",
            "https://ai2-s2-scispacy.s3-us-west-2.amazonaws.com/data/linkers/2023-04-23/go/tfidf_vectorizer.joblib not found in cache, downloading to /tmp/tmpd61y25yf\n",
            "Finished download, copying /tmp/tmpd61y25yf to cache at /root/.scispacy/datasets/2be1d8fd599f1a1d6140e5af989f4f512613b42a0b74a603aa599c038fc939b5.716eb257880cc667cc906fd73f357c3fccf3abec701904072f3b53b02a45deba.tfidf_vectorizer.joblib\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/sklearn/base.py:318: UserWarning: Trying to unpickle estimator TfidfTransformer from version 1.1.2 when using version 1.2.2. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\n",
            "https://scikit-learn.org/stable/model_persistence.html#security-maintainability-limitations\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/sklearn/base.py:318: UserWarning: Trying to unpickle estimator TfidfVectorizer from version 1.1.2 when using version 1.2.2. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\n",
            "https://scikit-learn.org/stable/model_persistence.html#security-maintainability-limitations\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "https://ai2-s2-scispacy.s3-us-west-2.amazonaws.com/data/linkers/2023-04-23/go/concept_aliases.json not found in cache, downloading to /tmp/tmpo1727nh5\n",
            "Finished download, copying /tmp/tmpo1727nh5 to cache at /root/.scispacy/datasets/e4e99357becdaacb55a07f8b1bcee8d7f6a634ab41db03ab28182a2166f24d4c.5b185b2b9139bc990299750dd4c87979e814ffee13ae3c650bc218c96dbc63ae.concept_aliases.json\n",
            "https://ai2-s2-scispacy.s3-us-west-2.amazonaws.com/data/kbs/2023-04-23/umls_go_2022.jsonl not found in cache, downloading to /tmp/tmpssoff2yd\n",
            "Finished download, copying /tmp/tmpssoff2yd to cache at /root/.scispacy/datasets/f2fae68affc838ddf0a87884154533ce359bda3c7d430bb7aa21ae851bee639d.0f776e01d8c81b2c7a6b9b8ffeff2bd7dc23c2b06fdc7719513bd10f1cff9c5a.umls_go_2022.jsonl\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "entities_dataframe['go_output'] = entities_dataframe['Entity'].apply(lambda x : go_entity_linker(x))"
      ],
      "metadata": {
        "id": "YFeGfr07Hmug"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "entities_dataframe"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 606
        },
        "id": "cyUXddEIHpCM",
        "outputId": "e85402e0-dc09-4145-e4c0-3591e97a9620"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                       Entity                 Label   Ner_model  \\\n",
              "0                    patients              ORGANISM  bionlp13cg   \n",
              "1    loperamide hydrochloride       SIMPLE_CHEMICAL  bionlp13cg   \n",
              "2             sodium chloride       SIMPLE_CHEMICAL  bionlp13cg   \n",
              "3                   gut-liver    CELLULAR_COMPONENT  bionlp13cg   \n",
              "4                 lymphocytes                  CELL  bionlp13cg   \n",
              "..                        ...                   ...         ...   \n",
              "96               electrolytes    CELLULAR_COMPONENT  bionlp13cg   \n",
              "97                 lymphocyte                  CELL  bionlp13cg   \n",
              "98         C-reactive protein  GENE_OR_GENE_PRODUCT  bionlp13cg   \n",
              "99                glutathione       SIMPLE_CHEMICAL  bionlp13cg   \n",
              "100                C-reactive  GENE_OR_GENE_PRODUCT  bionlp13cg   \n",
              "\n",
              "                                           mesh_output  \\\n",
              "0    [(patients), Entity_Matching_Score :1.0, (C003...   \n",
              "1    [(loperamide, hydrochloride), Entity_Matching_...   \n",
              "2    [(sodium, chloride), Entity_Matching_Score :1....   \n",
              "3                                        [(gut-liver)]   \n",
              "4    [(lymphocytes), Entity_Matching_Score :1.0, (C...   \n",
              "..                                                 ...   \n",
              "96   [(electrolytes), Entity_Matching_Score :1.0, (...   \n",
              "97   [(lymphocyte), Entity_Matching_Score :1.0, (C0...   \n",
              "98   [(C-reactive, protein), Entity_Matching_Score ...   \n",
              "99   [(glutathione), Entity_Matching_Score :1.0, (C...   \n",
              "100                                              [Nan]   \n",
              "\n",
              "                                            hpo_output  \\\n",
              "0                                         [(patients)]   \n",
              "1                        [(loperamide, hydrochloride)]   \n",
              "2                                 [(sodium, chloride)]   \n",
              "3                                        [(gut-liver)]   \n",
              "4    [(lymphocytes), Entity_Matching_Score :0.90991...   \n",
              "..                                                 ...   \n",
              "96   [(electrolytes), Entity_Matching_Score :0.7037...   \n",
              "97   [(lymphocyte), Entity_Matching_Score :0.831573...   \n",
              "98   [(C-reactive, protein), Entity_Matching_Score ...   \n",
              "99                                     [(glutathione)]   \n",
              "100                                              [Nan]   \n",
              "\n",
              "                                         rxnorm_output  \\\n",
              "0                                         [(patients)]   \n",
              "1    [(loperamide, hydrochloride), Entity_Matching_...   \n",
              "2    [(sodium, chloride), Entity_Matching_Score :1....   \n",
              "3                                        [(gut-liver)]   \n",
              "4                                      [(lymphocytes)]   \n",
              "..                                                 ...   \n",
              "96                                    [(electrolytes)]   \n",
              "97                                      [(lymphocyte)]   \n",
              "98                             [(C-reactive, protein)]   \n",
              "99   [(glutathione), Entity_Matching_Score :1.0, (C...   \n",
              "100                                              [Nan]   \n",
              "\n",
              "                                             go_output  \n",
              "0                                         [(patients)]  \n",
              "1                        [(loperamide, hydrochloride)]  \n",
              "2                                 [(sodium, chloride)]  \n",
              "3                                        [(gut-liver)]  \n",
              "4    [(lymphocytes), Entity_Matching_Score :0.77936...  \n",
              "..                                                 ...  \n",
              "96                                    [(electrolytes)]  \n",
              "97   [(lymphocyte), Entity_Matching_Score :0.896024...  \n",
              "98                             [(C-reactive, protein)]  \n",
              "99   [(glutathione), Entity_Matching_Score :0.83615...  \n",
              "100                                              [Nan]  \n",
              "\n",
              "[101 rows x 7 columns]"
            ],
            "text/html": [
              "\n",
              "\n",
              "  <div id=\"df-756d4336-98f4-4e2a-9fbb-69fd54b54c01\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Entity</th>\n",
              "      <th>Label</th>\n",
              "      <th>Ner_model</th>\n",
              "      <th>mesh_output</th>\n",
              "      <th>hpo_output</th>\n",
              "      <th>rxnorm_output</th>\n",
              "      <th>go_output</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>patients</td>\n",
              "      <td>ORGANISM</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(patients), Entity_Matching_Score :1.0, (C003...</td>\n",
              "      <td>[(patients)]</td>\n",
              "      <td>[(patients)]</td>\n",
              "      <td>[(patients)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>loperamide hydrochloride</td>\n",
              "      <td>SIMPLE_CHEMICAL</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(loperamide, hydrochloride), Entity_Matching_...</td>\n",
              "      <td>[(loperamide, hydrochloride)]</td>\n",
              "      <td>[(loperamide, hydrochloride), Entity_Matching_...</td>\n",
              "      <td>[(loperamide, hydrochloride)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>sodium chloride</td>\n",
              "      <td>SIMPLE_CHEMICAL</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(sodium, chloride), Entity_Matching_Score :1....</td>\n",
              "      <td>[(sodium, chloride)]</td>\n",
              "      <td>[(sodium, chloride), Entity_Matching_Score :1....</td>\n",
              "      <td>[(sodium, chloride)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>gut-liver</td>\n",
              "      <td>CELLULAR_COMPONENT</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(gut-liver)]</td>\n",
              "      <td>[(gut-liver)]</td>\n",
              "      <td>[(gut-liver)]</td>\n",
              "      <td>[(gut-liver)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>lymphocytes</td>\n",
              "      <td>CELL</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(lymphocytes), Entity_Matching_Score :1.0, (C...</td>\n",
              "      <td>[(lymphocytes), Entity_Matching_Score :0.90991...</td>\n",
              "      <td>[(lymphocytes)]</td>\n",
              "      <td>[(lymphocytes), Entity_Matching_Score :0.77936...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>96</th>\n",
              "      <td>electrolytes</td>\n",
              "      <td>CELLULAR_COMPONENT</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(electrolytes), Entity_Matching_Score :1.0, (...</td>\n",
              "      <td>[(electrolytes), Entity_Matching_Score :0.7037...</td>\n",
              "      <td>[(electrolytes)]</td>\n",
              "      <td>[(electrolytes)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>97</th>\n",
              "      <td>lymphocyte</td>\n",
              "      <td>CELL</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(lymphocyte), Entity_Matching_Score :1.0, (C0...</td>\n",
              "      <td>[(lymphocyte), Entity_Matching_Score :0.831573...</td>\n",
              "      <td>[(lymphocyte)]</td>\n",
              "      <td>[(lymphocyte), Entity_Matching_Score :0.896024...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>98</th>\n",
              "      <td>C-reactive protein</td>\n",
              "      <td>GENE_OR_GENE_PRODUCT</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(C-reactive, protein), Entity_Matching_Score ...</td>\n",
              "      <td>[(C-reactive, protein), Entity_Matching_Score ...</td>\n",
              "      <td>[(C-reactive, protein)]</td>\n",
              "      <td>[(C-reactive, protein)]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>99</th>\n",
              "      <td>glutathione</td>\n",
              "      <td>SIMPLE_CHEMICAL</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[(glutathione), Entity_Matching_Score :1.0, (C...</td>\n",
              "      <td>[(glutathione)]</td>\n",
              "      <td>[(glutathione), Entity_Matching_Score :1.0, (C...</td>\n",
              "      <td>[(glutathione), Entity_Matching_Score :0.83615...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>100</th>\n",
              "      <td>C-reactive</td>\n",
              "      <td>GENE_OR_GENE_PRODUCT</td>\n",
              "      <td>bionlp13cg</td>\n",
              "      <td>[Nan]</td>\n",
              "      <td>[Nan]</td>\n",
              "      <td>[Nan]</td>\n",
              "      <td>[Nan]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>101 rows × 7 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-756d4336-98f4-4e2a-9fbb-69fd54b54c01')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "\n",
              "\n",
              "\n",
              "    <div id=\"df-ba824d36-eb01-4256-86ea-dd6afba9acda\">\n",
              "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-ba824d36-eb01-4256-86ea-dd6afba9acda')\"\n",
              "              title=\"Suggest charts.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "      </button>\n",
              "    </div>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "    <script>\n",
              "      async function quickchart(key) {\n",
              "        const containerElement = document.querySelector('#' + key);\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      }\n",
              "    </script>\n",
              "\n",
              "      <script>\n",
              "\n",
              "function displayQuickchartButton(domScope) {\n",
              "  let quickchartButtonEl =\n",
              "    domScope.querySelector('#df-ba824d36-eb01-4256-86ea-dd6afba9acda button.colab-df-quickchart');\n",
              "  quickchartButtonEl.style.display =\n",
              "    google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "}\n",
              "\n",
              "        displayQuickchartButton(document);\n",
              "      </script>\n",
              "      <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-756d4336-98f4-4e2a-9fbb-69fd54b54c01 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-756d4336-98f4-4e2a-9fbb-69fd54b54c01');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "entities_dataframe['go_output'][4]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eYJxZ-51HrCX",
        "outputId": "e7c346c1-546a-4dff-a506-803c9902dcab"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[lymphocytes,\n",
              " 'Entity_Matching_Score :0.7793624997138977',\n",
              " CUI: C1326202, Name: B cell apoptotic process\n",
              " Definition: Any apoptotic process in a B cell, a lymphocyte of B lineage with the phenotype CD19-positive and capable of B cell mediated immunity. [CL:0000236, GOC:add, GOC:mtg_apoptosis, ISBN:0781735149]\n",
              " TUI(s): T043\n",
              " Aliases (abbreviated, total: 20): \n",
              " \t B lymphocyte apoptosis, programmed cell death, B lymphocytes, apoptosis of B-lymphocytes, programmed cell death of B-lymphocytes by apoptosis, programmed cell death of B cells by apoptosis, apoptosis of B-cells, B cell programmed cell death by apoptosis, B-cell programmed cell death by apoptosis, programmed cell death, B cells, B-cell apoptosis,\n",
              " 'Entity_Matching_Score :0.7451572418212891',\n",
              " CUI: C0024262, Name: lymphocyte activation\n",
              " Definition: Morphologic alteration of small B LYMPHOCYTES or T LYMPHOCYTES in culture into large blast-like cells able to synthesize DNA and RNA and to divide mitotically. It is induced by INTERLEUKINS; MITOGENS such as PHYTOHEMAGGLUTININS, and by specific ANTIGENS. It may also occur in vivo as in GRAFT REJECTION.\n",
              " TUI(s): T043\n",
              " Aliases: (total: 0): \n",
              " \t ,\n",
              " 'Entity_Matching_Score :0.7069276571273804',\n",
              " CUI: C1155065, Name: T lymphocyte activation\n",
              " Definition: The change in morphology and behavior of a mature or immature T cell resulting from exposure to a mitogen, cytokine, chemokine, cellular ligand, or an antigen for which it is specific. [GOC:mgi_curators, ISBN:0781735149]\n",
              " TUI(s): T043\n",
              " Aliases: (total: 3): \n",
              " \t T-cell activation, T cell activation, T-lymphocyte activation,\n",
              " 'Entity_Matching_Score :0.7021284699440002',\n",
              " CUI: C1155003, Name: B lymphocyte activation\n",
              " Definition: The change in morphology and behavior of a mature or immature B cell resulting from exposure to a mitogen, cytokine, chemokine, cellular ligand, or an antigen for which it is specific. [GOC:mgi_curators, ISBN:0781735149]\n",
              " TUI(s): T043\n",
              " Aliases: (total: 3): \n",
              " \t B cell activation, B-cell activation, B-lymphocyte activation]"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    }
  ]
}