SG_CQG_Generate_Conversations.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WiftNY4tckW0",
        "outputId": "346a4115-db07-4957-9821-a4e5120a706a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from platform import python_version\n",
        "\n",
        "print(python_version())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Tan7IPdVenhy",
        "outputId": "a2382c30-98a1-4bcb-b2ff-e9303cc09d9a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "3.8.10\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install fuzzywuzzy\n",
        "!pip install networkx\n",
        "!pip install -U pip setuptools wheel\n",
        "!pip install -U spacy\n",
        "!python -m spacy download en_core_web_sm\n",
        "!apt-get install -y graphviz-dev\n",
        "!pip install pygraphviz"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CKdzTW-PcuUB",
        "outputId": "10f0a161-ed42-4611-aed2-f22ef73d36b1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting fuzzywuzzy\n",
            "  Downloading fuzzywuzzy-0.18.0-py2.py3-none-any.whl (18 kB)\n",
            "Installing collected packages: fuzzywuzzy\n",
            "Successfully installed fuzzywuzzy-0.18.0\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.8/dist-packages (3.0)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: pip in /usr/local/lib/python3.8/dist-packages (22.0.4)\n",
            "Collecting pip\n",
            "  Downloading pip-23.0.1-py3-none-any.whl (2.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m53.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: setuptools in /usr/local/lib/python3.8/dist-packages (57.4.0)\n",
            "Collecting setuptools\n",
            "  Downloading setuptools-67.4.0-py3-none-any.whl (1.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m71.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: wheel in /usr/local/lib/python3.8/dist-packages (0.38.4)\n",
            "Installing collected packages: setuptools, pip\n",
            "  Attempting uninstall: setuptools\n",
            "    Found existing installation: setuptools 57.4.0\n",
            "    Uninstalling setuptools-57.4.0:\n",
            "      Successfully uninstalled setuptools-57.4.0\n",
            "  Attempting uninstall: pip\n",
            "    Found existing installation: pip 22.0.4\n",
            "    Uninstalling pip-22.0.4:\n",
            "      Successfully uninstalled pip-22.0.4\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "ipython 7.9.0 requires jedi>=0.10, which is not installed.\n",
            "cvxpy 1.2.3 requires setuptools<=64.0.2, but you have setuptools 67.4.0 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed pip-23.0.1 setuptools-67.4.0\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: spacy in /usr/local/lib/python3.8/dist-packages (3.4.4)\n",
            "Collecting spacy\n",
            "  Downloading spacy-3.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (6.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.7/6.7 MB\u001b[0m \u001b[31m88.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: jinja2 in /usr/local/lib/python3.8/dist-packages (from spacy) (2.11.3)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.8/dist-packages (from spacy) (4.64.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.8/dist-packages (from spacy) (67.4.0)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.8/dist-packages (from spacy) (2.0.8)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.8/dist-packages (from spacy) (2.25.1)\n",
            "Requirement already satisfied: typer<0.8.0,>=0.3.0 in /usr/local/lib/python3.8/dist-packages (from spacy) (0.7.0)\n",
            "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /usr/local/lib/python3.8/dist-packages (from spacy) (0.10.1)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from spacy) (3.0.8)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.8/dist-packages (from spacy) (3.3.0)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.8/dist-packages (from spacy) (1.0.9)\n",
            "Requirement already satisfied: thinc<8.2.0,>=8.1.0 in /usr/local/lib/python3.8/dist-packages (from spacy) (8.1.7)\n",
            "Requirement already satisfied: pathy>=0.10.0 in /usr/local/lib/python3.8/dist-packages (from spacy) (0.10.1)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/dist-packages (from spacy) (23.0)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /usr/local/lib/python3.8/dist-packages (from spacy) (3.0.12)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.8/dist-packages (from spacy) (2.4.5)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.8/dist-packages (from spacy) (1.0.4)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4 in /usr/local/lib/python3.8/dist-packages (from spacy) (1.10.5)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from spacy) (2.0.7)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.8/dist-packages (from spacy) (1.22.4)\n",
            "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /usr/local/lib/python3.8/dist-packages (from spacy) (6.3.0)\n",
            "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.8/dist-packages (from pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4->spacy) (4.5.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2.10)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (4.0.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2022.12.7)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.8/dist-packages (from thinc<8.2.0,>=8.1.0->spacy) (0.7.9)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.8/dist-packages (from thinc<8.2.0,>=8.1.0->spacy) (0.0.4)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.8/dist-packages (from typer<0.8.0,>=0.3.0->spacy) (7.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.8/dist-packages (from jinja2->spacy) (2.0.1)\n",
            "Installing collected packages: spacy\n",
            "  Attempting uninstall: spacy\n",
            "    Found existing installation: spacy 3.4.4\n",
            "    Uninstalling spacy-3.4.4:\n",
            "      Successfully uninstalled spacy-3.4.4\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "en-core-web-sm 3.4.1 requires spacy<3.5.0,>=3.4.0, but you have spacy 3.5.0 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed spacy-3.5.0\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m2023-02-23 06:23:07.114755: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F FMA\n",
            "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2023-02-23 06:23:08.000962: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/lib64-nvidia\n",
            "2023-02-23 06:23:08.001075: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/lib64-nvidia\n",
            "2023-02-23 06:23:08.001095: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting en-core-web-sm==3.5.0\n",
            "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.5.0/en_core_web_sm-3.5.0-py3-none-any.whl (12.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.8/12.8 MB\u001b[0m \u001b[31m66.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: spacy<3.6.0,>=3.5.0 in /usr/local/lib/python3.8/dist-packages (from en-core-web-sm==3.5.0) (3.5.0)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.8/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (2.0.8)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4 in /usr/local/lib/python3.8/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (1.10.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (23.0)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (1.0.9)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (2.0.7)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /usr/local/lib/python3.8/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (3.0.12)\n",
            "Requirement already satisfied: pathy>=0.10.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (0.10.1)\n",
            "Requirement already satisfied: typer<0.8.0,>=0.3.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (0.7.0)\n",
            "Requirement already satisfied: thinc<8.2.0,>=8.1.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (8.1.7)\n",
            "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /usr/local/lib/python3.8/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (0.10.1)\n",
            "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /usr/local/lib/python3.8/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (6.3.0)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (1.0.4)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (3.3.0)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (3.0.8)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (1.22.4)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (4.64.1)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (2.25.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.8/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (2.11.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.8/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (67.4.0)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.8/dist-packages (from spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (2.4.5)\n",
            "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.8/dist-packages (from pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (4.5.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (2022.12.7)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (4.0.0)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.8/dist-packages (from thinc<8.2.0,>=8.1.0->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (0.0.4)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.8/dist-packages (from thinc<8.2.0,>=8.1.0->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (0.7.9)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.8/dist-packages (from typer<0.8.0,>=0.3.0->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (7.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.8/dist-packages (from jinja2->spacy<3.6.0,>=3.5.0->en-core-web-sm==3.5.0) (2.0.1)\n",
            "Installing collected packages: en-core-web-sm\n",
            "  Attempting uninstall: en-core-web-sm\n",
            "    Found existing installation: en-core-web-sm 3.4.1\n",
            "    Uninstalling en-core-web-sm-3.4.1:\n",
            "      Successfully uninstalled en-core-web-sm-3.4.1\n",
            "Successfully installed en-core-web-sm-3.5.0\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the package via spacy.load('en_core_web_sm')\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "Note, selecting 'libgraphviz-dev' instead of 'graphviz-dev'\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-510\n",
            "Use 'apt autoremove' to remove it.\n",
            "The following additional packages will be installed:\n",
            "  libgail-common libgail18 libgtk2.0-0 libgtk2.0-bin libgtk2.0-common\n",
            "  libgvc6-plugins-gtk libxdot4\n",
            "Suggested packages:\n",
            "  gvfs\n",
            "The following NEW packages will be installed:\n",
            "  libgail-common libgail18 libgraphviz-dev libgtk2.0-0 libgtk2.0-bin\n",
            "  libgtk2.0-common libgvc6-plugins-gtk libxdot4\n",
            "0 upgraded, 8 newly installed, 0 to remove and 21 not upgraded.\n",
            "Need to get 2,148 kB of archives.\n",
            "After this operation, 7,427 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 libgtk2.0-common all 2.24.32-4ubuntu4 [126 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 libgtk2.0-0 amd64 2.24.32-4ubuntu4 [1,791 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 libgail18 amd64 2.24.32-4ubuntu4 [14.7 kB]\n",
            "Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 libgail-common amd64 2.24.32-4ubuntu4 [116 kB]\n",
            "Get:5 http://archive.ubuntu.com/ubuntu focal/universe amd64 libxdot4 amd64 2.42.2-3build2 [15.4 kB]\n",
            "Get:6 http://archive.ubuntu.com/ubuntu focal/universe amd64 libgvc6-plugins-gtk amd64 2.42.2-3build2 [20.6 kB]\n",
            "Get:7 http://archive.ubuntu.com/ubuntu focal/universe amd64 libgraphviz-dev amd64 2.42.2-3build2 [57.2 kB]\n",
            "Get:8 http://archive.ubuntu.com/ubuntu focal/main amd64 libgtk2.0-bin amd64 2.24.32-4ubuntu4 [7,728 B]\n",
            "Fetched 2,148 kB in 0s (15.9 MB/s)\n",
            "Selecting previously unselected package libgtk2.0-common.\n",
            "(Reading database ... 128126 files and directories currently installed.)\n",
            "Preparing to unpack .../0-libgtk2.0-common_2.24.32-4ubuntu4_all.deb ...\n",
            "Unpacking libgtk2.0-common (2.24.32-4ubuntu4) ...\n",
            "Selecting previously unselected package libgtk2.0-0:amd64.\n",
            "Preparing to unpack .../1-libgtk2.0-0_2.24.32-4ubuntu4_amd64.deb ...\n",
            "Unpacking libgtk2.0-0:amd64 (2.24.32-4ubuntu4) ...\n",
            "Selecting previously unselected package libgail18:amd64.\n",
            "Preparing to unpack .../2-libgail18_2.24.32-4ubuntu4_amd64.deb ...\n",
            "Unpacking libgail18:amd64 (2.24.32-4ubuntu4) ...\n",
            "Selecting previously unselected package libgail-common:amd64.\n",
            "Preparing to unpack .../3-libgail-common_2.24.32-4ubuntu4_amd64.deb ...\n",
            "Unpacking libgail-common:amd64 (2.24.32-4ubuntu4) ...\n",
            "Selecting previously unselected package libxdot4:amd64.\n",
            "Preparing to unpack .../4-libxdot4_2.42.2-3build2_amd64.deb ...\n",
            "Unpacking libxdot4:amd64 (2.42.2-3build2) ...\n",
            "Selecting previously unselected package libgvc6-plugins-gtk.\n",
            "Preparing to unpack .../5-libgvc6-plugins-gtk_2.42.2-3build2_amd64.deb ...\n",
            "Unpacking libgvc6-plugins-gtk (2.42.2-3build2) ...\n",
            "Selecting previously unselected package libgraphviz-dev:amd64.\n",
            "Preparing to unpack .../6-libgraphviz-dev_2.42.2-3build2_amd64.deb ...\n",
            "Unpacking libgraphviz-dev:amd64 (2.42.2-3build2) ...\n",
            "Selecting previously unselected package libgtk2.0-bin.\n",
            "Preparing to unpack .../7-libgtk2.0-bin_2.24.32-4ubuntu4_amd64.deb ...\n",
            "Unpacking libgtk2.0-bin (2.24.32-4ubuntu4) ...\n",
            "Setting up libxdot4:amd64 (2.42.2-3build2) ...\n",
            "Setting up libgtk2.0-common (2.24.32-4ubuntu4) ...\n",
            "Setting up libgtk2.0-0:amd64 (2.24.32-4ubuntu4) ...\n",
            "Setting up libgvc6-plugins-gtk (2.42.2-3build2) ...\n",
            "Setting up libgail18:amd64 (2.24.32-4ubuntu4) ...\n",
            "Setting up libgtk2.0-bin (2.24.32-4ubuntu4) ...\n",
            "Setting up libgail-common:amd64 (2.24.32-4ubuntu4) ...\n",
            "Setting up libgraphviz-dev:amd64 (2.42.2-3build2) ...\n",
            "Processing triggers for man-db (2.9.1-1) ...\n",
            "Processing triggers for libc-bin (2.31-0ubuntu9.9) ...\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting pygraphviz\n",
            "  Downloading pygraphviz-1.10.zip (120 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m120.6/120.6 kB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Building wheels for collected packages: pygraphviz\n",
            "  Building wheel for pygraphviz (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pygraphviz: filename=pygraphviz-1.10-cp38-cp38-linux_x86_64.whl size=184582 sha256=773e9b10060c535b63564861eef06a0bd570ac581f79a44cf60c90ab4386724b\n",
            "  Stored in directory: /root/.cache/pip/wheels/ac/bd/46/118c2a336c6fd8d47026854302f8fad63bb7e431e52008ea06\n",
            "Successfully built pygraphviz\n",
            "Installing collected packages: pygraphviz\n",
            "Successfully installed pygraphviz-1.10\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download('punkt')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O7SMU0Rnc0hC",
        "outputId": "01b0ff41-801e-44ec-9944-7be7d24a0f72"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cd fairseq"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rnGqFZ8hc2iX",
        "outputId": "1fabc3b2-728b-4a1b-e7be-d9fa174522ff"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[Errno 2] No such file or directory: 'fairseq'\n",
            "/content\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python setup.py build develop\n",
        "!pip install --editable ./"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BlpprFGYdDsd",
        "outputId": "d2b23af2-461b-4251-a7ed-4e776d6e77ff"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "python3: can't open file 'setup.py': [Errno 2] No such file or directory\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Obtaining file:///content\n",
            "\u001b[31mERROR: file:///content does not appear to be a Python project: neither 'setup.py' nor 'pyproject.toml' found.\u001b[0m\u001b[31m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h2> 1. Load QA and QG model"
      ],
      "metadata": {
        "id": "WW3agg7adeAL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%cd /content/gdrive/MyDrive/A/SG-CQG\n",
        "!pip install git+https://github.com/huggingface/transformers\n",
        "!pip install -r requirements.txt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "46aH9sn4dOXr",
        "outputId": "176a3b40-72a4-4853-ffe7-21526efbfc33"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/gdrive/MyDrive/A/SG-CQG\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting git+https://github.com/huggingface/transformers\n",
            "  Cloning https://github.com/huggingface/transformers to /tmp/pip-req-build-q1b_4hyx\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/huggingface/transformers /tmp/pip-req-build-q1b_4hyx\n",
            "  Resolved https://github.com/huggingface/transformers to commit df06fb1f0b0864ca0c2dcd9f7f6aab5de6447c59\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.8/dist-packages (from transformers==4.27.0.dev0) (4.64.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.8/dist-packages (from transformers==4.27.0.dev0) (6.0)\n",
            "Collecting huggingface-hub<1.0,>=0.11.0\n",
            "  Downloading huggingface_hub-0.12.1-py3-none-any.whl (190 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m190.3/190.3 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from transformers==4.27.0.dev0) (3.9.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.8/dist-packages (from transformers==4.27.0.dev0) (2.25.1)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/dist-packages (from transformers==4.27.0.dev0) (23.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.8/dist-packages (from transformers==4.27.0.dev0) (1.21.6)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.8/dist-packages (from transformers==4.27.0.dev0) (2022.6.2)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m48.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.8/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers==4.27.0.dev0) (4.5.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests->transformers==4.27.0.dev0) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests->transformers==4.27.0.dev0) (2022.12.7)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests->transformers==4.27.0.dev0) (2.10)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests->transformers==4.27.0.dev0) (4.0.0)\n",
            "Building wheels for collected packages: transformers\n",
            "  Building wheel for transformers (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for transformers: filename=transformers-4.27.0.dev0-py3-none-any.whl size=6623942 sha256=5391375ed57566b2d89ad022f7a5f9b1c2d2777280a49322ebdba1e3b3c02728\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-zg0nkzdb/wheels/42/68/45/c63edff61c292f2dfd4df4ef6522dcbecc603e7af82813c1d7\n",
            "Successfully built transformers\n",
            "Installing collected packages: tokenizers, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.12.1 tokenizers-0.13.2 transformers-4.27.0.dev0\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0mLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting accelerate\n",
            "  Downloading accelerate-0.16.0-py3-none-any.whl (199 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m199.7/199.7 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting datasets>=1.8.0\n",
            "  Downloading datasets-2.9.0-py3-none-any.whl (462 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m462.8/462.8 kB\u001b[0m \u001b[31m23.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting sentencepiece!=0.1.92\n",
            "  Downloading sentencepiece-0.1.97-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m53.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: protobuf in /usr/local/lib/python3.8/dist-packages (from -r requirements.txt (line 4)) (3.19.6)\n",
            "Collecting rouge-score\n",
            "  Downloading rouge_score-0.1.2.tar.gz (17 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.8/dist-packages (from -r requirements.txt (line 6)) (3.7)\n",
            "Collecting py7zr\n",
            "  Downloading py7zr-0.20.4-py3-none-any.whl (66 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m66.3/66.3 kB\u001b[0m \u001b[31m10.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: torch>=1.3 in /usr/local/lib/python3.8/dist-packages (from -r requirements.txt (line 8)) (1.13.1+cu116)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.8/dist-packages (from accelerate->-r requirements.txt (line 1)) (1.21.6)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.8/dist-packages (from accelerate->-r requirements.txt (line 1)) (6.0)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.8/dist-packages (from accelerate->-r requirements.txt (line 1)) (5.4.8)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/dist-packages (from accelerate->-r requirements.txt (line 1)) (23.0)\n",
            "Requirement already satisfied: dill<0.3.7 in /usr/local/lib/python3.8/dist-packages (from datasets>=1.8.0->-r requirements.txt (line 2)) (0.3.6)\n",
            "Collecting xxhash\n",
            "  Downloading xxhash-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (213 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m213.0/213.0 kB\u001b[0m \u001b[31m26.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: huggingface-hub<1.0.0,>=0.2.0 in /usr/local/lib/python3.8/dist-packages (from datasets>=1.8.0->-r requirements.txt (line 2)) (0.12.1)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.8/dist-packages (from datasets>=1.8.0->-r requirements.txt (line 2)) (3.8.4)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.8/dist-packages (from datasets>=1.8.0->-r requirements.txt (line 2)) (1.3.5)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.8/dist-packages (from datasets>=1.8.0->-r requirements.txt (line 2)) (2.25.1)\n",
            "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.8/dist-packages (from datasets>=1.8.0->-r requirements.txt (line 2)) (9.0.0)\n",
            "Collecting responses<0.19\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.8/dist-packages (from datasets>=1.8.0->-r requirements.txt (line 2)) (2023.1.0)\n",
            "Collecting multiprocess\n",
            "  Downloading multiprocess-0.70.14-py38-none-any.whl (132 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.0/132.0 kB\u001b[0m \u001b[31m19.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.8/dist-packages (from datasets>=1.8.0->-r requirements.txt (line 2)) (4.64.1)\n",
            "Requirement already satisfied: absl-py in /usr/local/lib/python3.8/dist-packages (from rouge-score->-r requirements.txt (line 5)) (1.4.0)\n",
            "Requirement already satisfied: six>=1.14.0 in /usr/local/lib/python3.8/dist-packages (from rouge-score->-r requirements.txt (line 5)) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.8/dist-packages (from nltk->-r requirements.txt (line 6)) (1.2.0)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.8/dist-packages (from nltk->-r requirements.txt (line 6)) (2022.6.2)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.8/dist-packages (from nltk->-r requirements.txt (line 6)) (7.1.2)\n",
            "Collecting brotli>=1.0.9\n",
            "  Downloading Brotli-1.0.9-cp38-cp38-manylinux1_x86_64.whl (357 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m357.2/357.2 kB\u001b[0m \u001b[31m41.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting multivolumefile>=0.2.3\n",
            "  Downloading multivolumefile-0.2.3-py3-none-any.whl (17 kB)\n",
            "Collecting pyzstd>=0.14.4\n",
            "  Downloading pyzstd-0.15.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (378 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m379.0/379.0 kB\u001b[0m \u001b[31m28.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting texttable\n",
            "  Downloading texttable-1.6.7-py2.py3-none-any.whl (10 kB)\n",
            "Collecting pybcj>=0.6.0\n",
            "  Downloading pybcj-1.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (50 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.8/50.8 kB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pycryptodomex>=3.6.6\n",
            "  Downloading pycryptodomex-3.17-cp35-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m76.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting inflate64>=0.3.1\n",
            "  Downloading inflate64-0.3.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (94 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m94.5/94.5 kB\u001b[0m \u001b[31m14.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pyppmd<1.1.0,>=0.18.1\n",
            "  Downloading pyppmd-1.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (139 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.7/139.7 kB\u001b[0m \u001b[31m17.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.8/dist-packages (from torch>=1.3->-r requirements.txt (line 8)) (4.5.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets>=1.8.0->-r requirements.txt (line 2)) (1.3.1)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets>=1.8.0->-r requirements.txt (line 2)) (4.0.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets>=1.8.0->-r requirements.txt (line 2)) (1.3.3)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets>=1.8.0->-r requirements.txt (line 2)) (22.2.0)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets>=1.8.0->-r requirements.txt (line 2)) (1.8.2)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets>=1.8.0->-r requirements.txt (line 2)) (3.0.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets>=1.8.0->-r requirements.txt (line 2)) (6.0.4)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets>=1.8.0->-r requirements.txt (line 2)) (3.9.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests>=2.19.0->datasets>=1.8.0->-r requirements.txt (line 2)) (2022.12.7)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests>=2.19.0->datasets>=1.8.0->-r requirements.txt (line 2)) (1.24.3)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests>=2.19.0->datasets>=1.8.0->-r requirements.txt (line 2)) (4.0.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests>=2.19.0->datasets>=1.8.0->-r requirements.txt (line 2)) (2.10)\n",
            "Collecting urllib3<1.27,>=1.21.1\n",
            "  Downloading urllib3-1.26.14-py2.py3-none-any.whl (140 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m140.6/140.6 kB\u001b[0m \u001b[31m19.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.8/dist-packages (from pandas->datasets>=1.8.0->-r requirements.txt (line 2)) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.8/dist-packages (from pandas->datasets>=1.8.0->-r requirements.txt (line 2)) (2022.7.1)\n",
            "Building wheels for collected packages: rouge-score\n",
            "  Building wheel for rouge-score (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for rouge-score: filename=rouge_score-0.1.2-py3-none-any.whl size=24935 sha256=6eff3322838a36f81fe4cd5d9f7233d2adbb902e8640ee2cbdf7885212367a0d\n",
            "  Stored in directory: /root/.cache/pip/wheels/24/55/6f/ebfc4cb176d1c9665da4e306e1705496206d08215c1acd9dde\n",
            "Successfully built rouge-score\n",
            "Installing collected packages: texttable, sentencepiece, brotli, xxhash, urllib3, pyzstd, pyppmd, pycryptodomex, pybcj, multivolumefile, multiprocess, inflate64, rouge-score, py7zr, accelerate, responses, datasets\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 1.24.3\n",
            "    Uninstalling urllib3-1.24.3:\n",
            "      Successfully uninstalled urllib3-1.24.3\n",
            "Successfully installed accelerate-0.16.0 brotli-1.0.9 datasets-2.9.0 inflate64-0.3.1 multiprocess-0.70.14 multivolumefile-0.2.3 py7zr-0.20.4 pybcj-1.0.1 pycryptodomex-3.17 pyppmd-1.0.0 pyzstd-0.15.3 responses-0.18.0 rouge-score-0.1.2 sentencepiece-0.1.97 texttable-1.6.7 urllib3-1.26.14 xxhash-3.2.0\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import T5Tokenizer, T5Config, T5ForConditionalGeneration\n",
        "import os\n",
        "from fuzzywuzzy import fuzz\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk import tokenize\n",
        "import random\n",
        "import networkx as nx\n",
        "import statistics\n",
        "from statistics import mode\n",
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "from nltk.corpus import stopwords\n",
        "import numpy as np\n",
        "\n",
        "stop_words = set(stopwords.words('english'))\n",
        "stop_words.add('.')\n",
        "stop_words.add('?')"
      ],
      "metadata": {
        "id": "SxwveSBletjv",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 373
        },
        "outputId": "88d763c8-243b-4d37-86e5-c621bc202bf7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ModuleNotFoundError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-1-8f603229cc71>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mtransformers\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mT5Tokenizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mT5Config\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mT5ForConditionalGeneration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mfuzzywuzzy\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mfuzz\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mnltk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenize\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mword_tokenize\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mnltk\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtokenize\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'transformers'",
            "",
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
          ],
          "errorDetails": {
            "actions": [
              {
                "action": "open_url",
                "actionText": "Open Examples",
                "url": "/notebooks/snippets/importing_libraries.ipynb"
              }
            ]
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cd /content/gdrive/MyDrive/A/SG-CQG/fairseq"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KKb7eFidFnmb",
        "outputId": "037c3589-90d2-4d11-ab97-e69290b624f4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/gdrive/MyDrive/A/SG-CQG/fairseq\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from fairseq.models.roberta import RobertaModel"
      ],
      "metadata": {
        "id": "x52yL0LRFRAx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class Generate_Conversation_from_Graph():\n",
        "  def __init__(self):\n",
        "    self.QA_MODEL_PATH = '/content/gdrive/MyDrive/A/factoid_one_focus/CoQA/T5-BART/output-T5-QA-max-3-turns/checkpoint-51000/'\n",
        "    self.QG_MODEL_PATH = '/content/gdrive/MyDrive/A/factoid_one_focus/CoQA/Answer-unaware/PromptT5/output-T5-CoQA/checkpoint-55000/'\n",
        "    self.KE_in_sentence_PATH = '/content/gdrive/MyDrive/A/factoid_one_focus/CoQA/T5-BART/KE-in-sentence/output-T5/checkpoint-48000/'\n",
        "\n",
        "    self.CLASSIFIER_PATH = \"/content/gdrive/MyDrive/A/factoid_one_focus/CoQA/Answer-unaware/Question-type-classifier/checkpoints\"\n",
        "    self.DATA_BIN_PATH = '/content/gdrive/MyDrive/A/factoid_one_focus/CoQA/Answer-unaware/Question-type-classifier/data-bin'\n",
        "    self.COLORS = {1: '#1E00F0',\n",
        "                  2: '#22182E',\n",
        "                  3: '#89DCC6',\n",
        "                  4: '#01F21D',\n",
        "                  5: '#1815C5',\n",
        "                  6: '#46D012',\n",
        "                  7: '#B0CCA9',\n",
        "                  8: '#943C5B',\n",
        "                  9: '#070C96',\n",
        "                  10: '#6B7212',\n",
        "                  11: '#974E48',\n",
        "                  12: '#0CC610',\n",
        "                  13: '#E82858',\n",
        "                  14: '#77E7A0',\n",
        "                  15: '#E93B03',\n",
        "                  16: '#4995C9',\n",
        "                  17: '#3201B7',\n",
        "                  18: '#80BE97',\n",
        "                  19: '#01C604',\n",
        "                  20: '#A074A0',\n",
        "                  21: '#035C8D',\n",
        "                  22: '#EB22D0',\n",
        "                  23: '#8D489F',\n",
        "                  24: '#68B7A9',\n",
        "                  25: '#F232D7',\n",
        "                  26: '#A67027',\n",
        "                  27: '#EBB891',\n",
        "                  28: '#C7FAC1',\n",
        "                  29: '#427716',\n",
        "                  30: '#04735A',\n",
        "                  31: '#4F4F26',\n",
        "                  32: '#AA7C3B',\n",
        "                  33: '#D4E151',\n",
        "                  34: '#65DBC6',\n",
        "                  35: '#CA62DD',\n",
        "                  36: '#89DD99',\n",
        "                  37: '#06F81F',\n",
        "                  38: '#29AC30',\n",
        "                  39: '#E06332',\n",
        "                  40: '#201B0C',\n",
        "                  41: '#8324A0',\n",
        "                  42: '#6AB686',\n",
        "                  43: '#240E87',\n",
        "                  44: '#E7B5A5',\n",
        "                  45: '#5F644F',\n",
        "                  46: '#27C198',\n",
        "                  47: '#D13A0A',\n",
        "                  48: '#214F1D',\n",
        "                  49: '#327728',\n",
        "                  50: '#F8DEAD',\n",
        "                  51: '#50BA8B',\n",
        "                  52: '#F37969',\n",
        "                  53: '#524D81',\n",
        "                  54: '#3DEE78',\n",
        "                  55: '#A764A7',\n",
        "                  56: '#6F50DF',\n",
        "                  57: '#9C4FBD',\n",
        "                  58: '#241F54',\n",
        "                  59: '#EB2E98',\n",
        "                  60: '#CF9C86',\n",
        "                  61: '#5F0FB1',\n",
        "                  62: '#DA61F2',\n",
        "                  63: '#54B7A5',\n",
        "                  64: '#F1821E',\n",
        "                  65: '#8D8DD6',\n",
        "                  66: '#BB12D2',\n",
        "                  67: '#48B772',\n",
        "                  68: '#8FF143',\n",
        "                  69: '#851CE7',\n",
        "                  70: '#6D81AA',\n",
        "                  71: '#6DAE07',\n",
        "                  72: '#B0D194',\n",
        "                  73: '#75EFCA',\n",
        "                  74: '#BDA529',\n",
        "                  75: '#84AD5F',\n",
        "                  76: '#8DAF43',\n",
        "                  77: '#13537A',\n",
        "                  78: '#9455C5',\n",
        "                  79: '#C5CF28',\n",
        "                  80: '#FE4BEB',\n",
        "                  81: '#09F7D8',\n",
        "                  82: '#4B64F5',\n",
        "                  83: '#B8C98F',\n",
        "                  84: '#ADE234',\n",
        "                  85: '#98F3B7',\n",
        "                  86: '#72344F',\n",
        "                  87: '#AC7940',\n",
        "                  88: '#021B4A',\n",
        "                  89: '#4C8A7C',\n",
        "                  90: '#CF9487',\n",
        "                  91: '#794BAE',\n",
        "                  92: '#C916AD',\n",
        "                  93: '#E9CF3D',\n",
        "                  94: '#666804',\n",
        "                  95: '#F31582',\n",
        "                  96: '#FD2DC1',\n",
        "                  97: '#C3E59A',\n",
        "                  98: '#B6F9E9',\n",
        "                  99: '#E14740',\n",
        "                  100: '#B3329E'}\n",
        "\n",
        "    self.QA_tokenizer, self.QA_config, self.QA_model = self.load_model_T5(self.QA_MODEL_PATH)\n",
        "    self.QG_tokenizer, self.QG_config, self.QG_model = self.load_model_T5(self.QG_MODEL_PATH)\n",
        "    self.KE_tokenizer, self.KE_config, self.KE_model = self.load_model_T5(self.KE_in_sentence_PATH)\n",
        "    self.roberta = RobertaModel.from_pretrained(\n",
        "      self.CLASSIFIER_PATH, checkpoint_file='checkpoint_best.pt',\n",
        "      data_name_or_path=self.DATA_BIN_PATH,\n",
        "    )\n",
        "\n",
        "  # Load model\n",
        "  def load_model_T5(self, MODEL_PATH):\n",
        "    tokenizer = T5Tokenizer.from_pretrained(MODEL_PATH)\n",
        "    config = T5Config.from_json_file(os.path.join(MODEL_PATH, 'config.json'))\n",
        "    model = T5ForConditionalGeneration.from_pretrained(MODEL_PATH + 'pytorch_model.bin', config=config)\n",
        "    return tokenizer, config, model\n",
        "\n",
        "  # Get QA answer\n",
        "  def get_QA_answer(self, question, context, history_string):\n",
        "    context = 'Question: ' + question + ' Context: ' + context + ' <s> ' + history_string\n",
        "    batch = self.QA_tokenizer(context, return_tensors='pt', max_length=1024, padding=\"max_length\", truncation=True)\n",
        "    generated_ids = self.QA_model.generate(\n",
        "      batch['input_ids'],\n",
        "      attention_mask=batch[\"attention_mask\"],\n",
        "      do_sample=True,\n",
        "      max_length=10,\n",
        "      top_p=0.92,\n",
        "      num_return_sequences=1\n",
        "    )\n",
        "    generated_answer = self.QA_tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]\n",
        "    return generated_answer\n",
        "\n",
        "  # Getting keyphrases from a sentence\n",
        "  def get_KEs_from_a_sentence(self, sent):\n",
        "    batch = self.KE_tokenizer(sent, return_tensors='pt', max_length=1024, padding=\"max_length\", truncation=True)\n",
        "    generated_ids = self.KE_model.generate(\n",
        "      batch['input_ids'],\n",
        "      attention_mask=batch[\"attention_mask\"],\n",
        "      do_sample=True,\n",
        "      max_length=10,\n",
        "      top_p=0.92,\n",
        "      num_return_sequences=3\n",
        "    )\n",
        "    KE_candidates = self.KE_tokenizer.batch_decode(generated_ids, skip_special_tokens=True)\n",
        "    KEs = []\n",
        "    for KE_sent in KE_candidates:\n",
        "      KEs.extend([ke.strip() for ke in KE_sent.split(' SEP>') if ke.strip() is not \"\"])\n",
        "\n",
        "    KEs = list(set(KEs))\n",
        "    KEs = [ke.strip() for ke in KEs if ke in sent]\n",
        "    final_KEs = []\n",
        "    for ke in KEs:\n",
        "      check = True\n",
        "      for ke_ in final_KEs:\n",
        "        if float(fuzz.ratio(ke.lower(), ke_.lower())) >= 80: check = False\n",
        "      if check: final_KEs.append(ke)\n",
        "    return final_KEs\n",
        "\n",
        "  # Get classification label\n",
        "  def get_prompt_token(self, test_classifier_input):\n",
        "    label_fn = lambda label: self.roberta.task.label_dictionary.string(\n",
        "      [label + self.roberta.task.label_dictionary.nspecial]\n",
        "    )\n",
        "    tokens = self.roberta.encode(test_classifier_input)\n",
        "    pred = label_fn(self.roberta.predict('imdb_head', tokens).argmax().item())\n",
        "    return '<BOOLEAN>' if str(pred) == '1' else '<NORMAL>'\n",
        "\n",
        "  def longestCommonSubstring(self, s, t):\n",
        "    n = len(s)\n",
        "    m = len(t)\n",
        "    dp = [[0 for i in range(m + 1)] for j in range(2)]\n",
        "    res = 0\n",
        "    for i in range(1,n + 1):\n",
        "      for j in range(1,m + 1):\n",
        "        if(s[i - 1] == t[j - 1]):\n",
        "          dp[i % 2][j] = dp[(i - 1) % 2][j - 1] + 1\n",
        "          if(dp[i % 2][j] > res):\n",
        "            res = dp[i % 2][j]\n",
        "        else:\n",
        "          dp[i % 2][j] = 0\n",
        "    return res\n",
        "\n",
        "  def get_sentence_index(self, rationale, sentences):\n",
        "    lengths = []\n",
        "    for idx in range(len(sentences)):\n",
        "      lengths.append(\n",
        "          self.longestCommonSubstring(rationale, sentences[idx])\n",
        "      )\n",
        "    return np.argmax(np.array(lengths)) + 1\n",
        "\n",
        "  def get_name_entities_and_coref_words(self, rationale, nodes_in_sentence, sentence_index):\n",
        "    if sentence_index not in nodes_in_sentence:\n",
        "      return []\n",
        "    nodes = []\n",
        "    for node in nodes_in_sentence[sentence_index]:\n",
        "      if node in rationale: nodes.append(node)\n",
        "\n",
        "    idx_entities = {}\n",
        "    for node in nodes:\n",
        "      idx_entities[node] = rationale.find(node)\n",
        "    sorted_nodes = [k for k, v in sorted(idx_entities.items(), key=lambda item: item[1])]\n",
        "    return sorted_nodes\n",
        "\n",
        "  def get_relevant_nodes_with_provided_history(self, rationales, sentences, nodes_in_sentence):\n",
        "    relevant_nodes = []\n",
        "    for rationale in rationales:\n",
        "      idx_sentence = self.get_sentence_index(rationale, sentences)\n",
        "      sorted_nodes = self.get_name_entities_and_coref_words(rationale,\n",
        "                                                      nodes_in_sentence,\n",
        "                                                      idx_sentence)\n",
        "      sorted_nodes.append(idx_sentence)\n",
        "      relevant_nodes.append(sorted_nodes)\n",
        "    return relevant_nodes\n",
        "\n",
        "  def get_relevant_nodes_without_provided_history(self, sentences, nodes_in_sentence):\n",
        "    relevant_nodes = []\n",
        "    idx = 1\n",
        "    while idx not in nodes_in_sentence: idx += 1\n",
        "\n",
        "    retrieved_sentence = sentences[idx-1]\n",
        "    relevant_nodes = nodes_in_sentence[idx]\n",
        "    idx_entities = {}\n",
        "    for node in relevant_nodes:\n",
        "      idx_entities[node] = retrieved_sentence.find(node)\n",
        "    sorted_nodes = [k for k, v in sorted(idx_entities.items(), key=lambda item: item[1])]\n",
        "\n",
        "    random.shuffle(sorted_nodes)\n",
        "    sorted_nodes.append(str(idx))\n",
        "    return [sorted_nodes]\n",
        "\n",
        "  def sentence_index_from_node(self, node_name):\n",
        "    return int(node_name[-2:].strip())\n",
        "\n",
        "  def get_node_name_from_node(self, node_name):\n",
        "    return str(node_name[:-2].strip())\n",
        "\n",
        "  def get_nodes_in_sentence(self, graph_triples):\n",
        "    nodes_in_sentence = {}\n",
        "    for triple in graph_triples:\n",
        "      head = list(triple)[0]\n",
        "      tail = list(triple)[1]\n",
        "\n",
        "      head_idx = self.sentence_index_from_node(head)\n",
        "      tail_idx = self.sentence_index_from_node(tail)\n",
        "      if head_idx not in nodes_in_sentence: nodes_in_sentence[head_idx] = []\n",
        "      if tail_idx not in nodes_in_sentence: nodes_in_sentence[tail_idx] = []\n",
        "\n",
        "      if self.get_node_name_from_node(head) not in nodes_in_sentence[head_idx]:\n",
        "        nodes_in_sentence[head_idx].append(self.get_node_name_from_node(head))\n",
        "      if self.get_node_name_from_node(tail) not in nodes_in_sentence[tail_idx]:\n",
        "        nodes_in_sentence[tail_idx].append(self.get_node_name_from_node(tail))\n",
        "    return nodes_in_sentence\n",
        "\n",
        "  def contruct_networkx_graph(self, context, graph_triples, test_ID):\n",
        "    G = nx.Graph()\n",
        "    for triple in graph_triples:\n",
        "      head = list(triple)[0]\n",
        "      tail = list(triple)[1]\n",
        "      attribute = list(triple)[2]\n",
        "\n",
        "      head_idx = self.sentence_index_from_node(head)\n",
        "      tail_idx = self.sentence_index_from_node(tail)\n",
        "      G.add_node(head, color = self.COLORS[int(head_idx)])\n",
        "      G.add_node(tail, color = self.COLORS[int(tail_idx)])\n",
        "\n",
        "      if 'label' in attribute: G.add_edge(head, tail, label=attribute['label'])\n",
        "      else: G.add_edge(head, tail)\n",
        "\n",
        "    A = nx.nx_agraph.to_agraph(G)\n",
        "    A.layout(prog='fdp')\n",
        "    nx.nx_agraph.write_dot(G, 'my_graph.dot')\n",
        "    A.draw(f\"/content/test_{str(test_ID)}.png\")\n",
        "    return G\n",
        "\n",
        "  def find_final_boolean_answer(self, hypothesised_answers):\n",
        "    hypothesised_answers = [i.lower() for i in hypothesised_answers]\n",
        "    try: return mode(hypothesised_answers)\n",
        "    except: return None\n",
        "\n",
        "  def verify_generated_question(self, signal_class, question_candidates, candidate_answer,\n",
        "                              context, sent, prev_answers, prev_questions, generated_qa_pairs):\n",
        "    hypothesised_answers = []\n",
        "    if len(generated_qa_pairs) <= 6: history_string = ' <s> '.join(generated_qa_pairs)\n",
        "    else: history_string = ' <s> '.join(generated_qa_pairs[-6:])\n",
        "\n",
        "    for candidate in question_candidates:\n",
        "      hypothesised_answers.append(self.get_QA_answer(candidate, sent + ' ' + context, history_string))\n",
        "\n",
        "    assert len(question_candidates) == len(hypothesised_answers)\n",
        "    print(question_candidates)\n",
        "    print(hypothesised_answers)\n",
        "\n",
        "    flag = False\n",
        "    if signal_class == '<BOOLEAN>':\n",
        "      for idx in range(len(question_candidates)):\n",
        "        question_candidate = question_candidates[idx]\n",
        "        if self.verify_question(question_candidate, prev_questions):\n",
        "          hypothesized_answer = self.find_final_boolean_answer(hypothesised_answers)\n",
        "          if hypothesized_answer is None: continue\n",
        "          generated_qa_pairs.extend([question_candidate, hypothesized_answer])\n",
        "          flag = True\n",
        "          return flag, generated_qa_pairs, hypothesized_answer\n",
        "      return flag, generated_qa_pairs, candidate_answer\n",
        "    else:\n",
        "      for idx in range(len(question_candidates)):\n",
        "        question_candidate = question_candidates[idx]\n",
        "        hypothesized_answer = hypothesised_answers[idx]\n",
        "        if self.verify_answer(hypothesized_answer, candidate_answer, prev_answers) and self.verify_question(question_candidate, prev_questions):\n",
        "          generated_qa_pairs.extend([question_candidate, hypothesized_answer])\n",
        "          flag = True\n",
        "          break\n",
        "      return flag, generated_qa_pairs, hypothesized_answer\n",
        "\n",
        "  def verify_question(self, current_question, prev_questions):\n",
        "    for pre_q in prev_questions:\n",
        "      if len(pre_q.split(\" \")) >= 4 and float(fuzz.ratio(pre_q.lower(), current_question.lower())) >= 80:\n",
        "        return False\n",
        "    return True\n",
        "\n",
        "  def verify_answer(self, hypothesized_answer, target_anwer, prev_answers):\n",
        "    if float(fuzz.ratio(hypothesized_answer.lower(), target_anwer.lower())) >= 80:\n",
        "      if self.check_with_previous_answers(hypothesized_answer, prev_answers):\n",
        "        return True\n",
        "    return False\n",
        "\n",
        "  def check_with_previous_answers(self, current_answer, prev_answers):\n",
        "    for pre_ans in prev_answers:\n",
        "      ratio_ = fuzz.ratio(pre_ans.lower(), current_answer.lower())\n",
        "      if float(ratio_) >= 80:\n",
        "        return False\n",
        "    return True\n",
        "\n",
        "  def generating_conversation(self, next_attempt_sentence_index, sent, sentences, context, generated_qa_pairs,\n",
        "                              prev_answers = set(), prev_questions = set(), previous_turn_tokens = set(),\n",
        "                              accepted_indexes = [], max_turn = 3, context_tokens = set()):\n",
        "    try: answers = self.get_KEs_from_a_sentence(sent)\n",
        "    except: return False\n",
        "\n",
        "    flag = False\n",
        "    for candidate_answer in answers:\n",
        "      full_history = ' <s> '.join(generated_qa_pairs)\n",
        "      # Max 3 previous turns\n",
        "      if len(generated_qa_pairs) <= 6: QA_pairs = ' <s> '.join(generated_qa_pairs)\n",
        "      else: QA_pairs = ' <s> '.join(generated_qa_pairs[-6:])\n",
        "\n",
        "      classifier_input = 'Answer: ' + candidate_answer + ' ' + sent + ' Context: ' + context + ' <SEP> ' + QA_pairs\n",
        "      signal_class = self.get_prompt_token(classifier_input)\n",
        "\n",
        "      prompt_QG_input = signal_class + ' ' + 'Answer: ' + candidate_answer + ' ' + sent + ' Context: ' + context + ' <SEP> ' + QA_pairs\n",
        "\n",
        "      batch = self.QG_tokenizer(prompt_QG_input, return_tensors='pt', max_length=1024, padding=\"max_length\", truncation=True)\n",
        "      generated_ids = self.QG_model.generate(\n",
        "        batch['input_ids'],\n",
        "        attention_mask=batch[\"attention_mask\"],\n",
        "        do_sample=True,\n",
        "        max_length=150,\n",
        "        top_p=0.92,\n",
        "        # beam_size=4,\n",
        "        num_return_sequences=3\n",
        "      )\n",
        "\n",
        "      question_candidates = self.QG_tokenizer.batch_decode(generated_ids, skip_special_tokens=True)\n",
        "      flag, generated_qa_pairs, hypothesized_answer = self.verify_generated_question(\n",
        "          signal_class,\n",
        "          question_candidates,\n",
        "          candidate_answer,\n",
        "          context,\n",
        "          sent,\n",
        "          prev_answers,\n",
        "          prev_questions,\n",
        "          generated_qa_pairs\n",
        "      )\n",
        "\n",
        "      if flag and hypothesized_answer in generated_qa_pairs[-2]:\n",
        "        flag = False\n",
        "        accepted_question = generated_qa_pairs[-2]\n",
        "        generated_qa_pairs.pop()\n",
        "        generated_qa_pairs.pop()\n",
        "        # print(\"Fail check answer not in question!\", str(hypothesized_answer), str(accepted_question))\n",
        "\n",
        "      if flag and signal_class == '<BOOLEAN>':\n",
        "        import random\n",
        "        drop_or_not = random.choice([1, 1])\n",
        "        accepted_question = generated_qa_pairs[-2]\n",
        "        if drop_or_not == 1:\n",
        "          if self.check_if_the_information_is_present_in_context(True, \"\", accepted_question, context_tokens):\n",
        "            prev_answers.add(hypothesized_answer)\n",
        "            prev_questions.add(generated_qa_pairs[-2])\n",
        "            accepted_indexes.append(next_attempt_sentence_index)\n",
        "\n",
        "            accepted_question = generated_qa_pairs[-2]\n",
        "            from nltk.tokenize import word_tokenize\n",
        "            for tok in word_tokenize(accepted_question): previous_turn_tokens.add(tok)\n",
        "          else:\n",
        "            generated_qa_pairs.pop()\n",
        "            generated_qa_pairs.pop()\n",
        "            flag = False\n",
        "        else:\n",
        "          generated_qa_pairs.pop()\n",
        "          generated_qa_pairs.pop()\n",
        "          flag = False\n",
        "\n",
        "      elif flag and signal_class == '<NORMAL>':\n",
        "        # accepted_indexes.append(next_attempt_sentence_index)\n",
        "        accepted_answer = hypothesized_answer\n",
        "        accepted_question = generated_qa_pairs[-2]\n",
        "\n",
        "        if self.check_redundant_information(accepted_answer, previous_turn_tokens):\n",
        "          # if check_if_the_information_is_present_in_context(True, accepted_answer, accepted_question, context_tokens):\n",
        "          prev_answers.add(hypothesized_answer)\n",
        "          prev_questions.add(generated_qa_pairs[-2])\n",
        "          accepted_indexes.append(next_attempt_sentence_index)\n",
        "          from nltk.tokenize import word_tokenize\n",
        "          for tok in word_tokenize(accepted_answer): previous_turn_tokens.add(tok)\n",
        "          for tok in word_tokenize(accepted_question): previous_turn_tokens.add(tok)\n",
        "        else:\n",
        "          generated_qa_pairs.pop()\n",
        "          generated_qa_pairs.pop()\n",
        "          # print(\"Fail check redundancy!, \", str(accepted_answer), str(previous_turn_tokens))\n",
        "          flag = False\n",
        "      else:\n",
        "        # print('FAIL CHECK: ', str(candidate_answer))\n",
        "        pass\n",
        "    return flag, prev_answers, prev_questions, generated_qa_pairs, previous_turn_tokens, accepted_indexes\n",
        "\n",
        "  def check_redundant_information(self, accepted_answer, previous_turn_tokens):\n",
        "    total = len(word_tokenize(accepted_answer))\n",
        "    cnt = 0\n",
        "    for tok in word_tokenize(accepted_answer):\n",
        "      if tok in previous_turn_tokens:\n",
        "        cnt += 1\n",
        "    if float(cnt/total) >= 0.5:\n",
        "      return False\n",
        "    return True\n",
        "\n",
        "  def remove_stop_words(self, example_sent):\n",
        "    word_tokens = word_tokenize(example_sent)\n",
        "    filtered_sentence = [w for w in word_tokens if not w.lower() in stop_words]\n",
        "    return filtered_sentence\n",
        "\n",
        "  def check_if_the_information_is_present_in_context(self, boolean_or_not, accepted_answer, accepted_question, context_tokens):\n",
        "    if boolean_or_not:\n",
        "      processed_ques = self.remove_stop_words(accepted_question)\n",
        "      for tok in processed_ques:\n",
        "        if tok not in context_tokens: return False\n",
        "    else:\n",
        "      processed_ans = self.remove_stop_words(accepted_answer)\n",
        "      processed_ques = self.remove_stop_words(accepted_question)\n",
        "      for tok in processed_ans:\n",
        "        if tok not in context_tokens: return False\n",
        "      for tok in processed_ques:\n",
        "        if tok not in context_tokens: return False\n",
        "    return True\n",
        "\n",
        "  def generate_full_conversation(self, context, graph_triples, test_ID):\n",
        "    sentences = tokenize.sent_tokenize(context)\n",
        "    nodes_in_sentence = self.get_nodes_in_sentence(graph_triples)\n",
        "\n",
        "    # relevant_nodes = list(reversed(get_relevant_nodes_with_provided_history(rationales, sentences, nodes_in_sentence)))\n",
        "    relevant_nodes = list(self.get_relevant_nodes_without_provided_history(sentences, nodes_in_sentence))\n",
        "\n",
        "    G = self.contruct_networkx_graph(context, graph_triples, test_ID)\n",
        "    print(\"Done graph initialization!\")\n",
        "\n",
        "    prev_answers = set()\n",
        "    prev_questions = set()\n",
        "    accepted_indexes = []\n",
        "    previous_turn_tokens = set()\n",
        "\n",
        "    conversational_history = \"\"\n",
        "    generated_qa_pairs = conversational_history.split(' <s> ')\n",
        "    context_tokens = set(word_tokenize(context))\n",
        "\n",
        "    marked_node = {}\n",
        "    marked_again = {}\n",
        "    for node in list(G.nodes):\n",
        "      marked_node[node] = 0\n",
        "      marked_again[node] = 1\n",
        "\n",
        "    queue = []\n",
        "    for relevant_set in relevant_nodes:\n",
        "      if len(relevant_set) == 1:\n",
        "        continue\n",
        "      sentence_idx = relevant_set[len(relevant_set)-1]\n",
        "      for node_idx in range(len(relevant_set)-1):\n",
        "        queue.append(relevant_set[node_idx] + \" \" + str(sentence_idx))\n",
        "\n",
        "    CONTINUE_SIGNAL = True\n",
        "    while(CONTINUE_SIGNAL):\n",
        "      if len(queue) == 0:\n",
        "        CONTINUE_SIGNAL = False\n",
        "        break\n",
        "\n",
        "      nearest_node = queue[0]\n",
        "      del queue[0]\n",
        "\n",
        "      if marked_node[nearest_node] == 1 and marked_again[nearest_node] == 1:\n",
        "        continue\n",
        "\n",
        "      next_node = None\n",
        "\n",
        "      if not marked_node[nearest_node]:\n",
        "        marked_node[nearest_node] = 1\n",
        "      elif marked_again[nearest_node] == 0:\n",
        "        marked_again[nearest_node] = 1\n",
        "      else:\n",
        "        continue\n",
        "\n",
        "      next_attempt_sentence_index = self.sentence_index_from_node(nearest_node)\n",
        "\n",
        "      SUCCESSFUL_FLAG, prev_answers, prev_questions, generated_qa_pairs, previous_turn_tokens, accepted_indexes = self.generating_conversation(\n",
        "          next_attempt_sentence_index - 1,\n",
        "          sentences[next_attempt_sentence_index - 1], sentences, context,\n",
        "          generated_qa_pairs, prev_answers, prev_questions, previous_turn_tokens, accepted_indexes, context_tokens\n",
        "      )\n",
        "\n",
        "      if SUCCESSFUL_FLAG:\n",
        "        neighbors_of_nearest_node = [n for n in G.neighbors(nearest_node)]\n",
        "        for node_ in neighbors_of_nearest_node:\n",
        "          if marked_node[node_] == 1 and marked_again[node_] == 1: continue\n",
        "          else: queue.insert(0, node_)\n",
        "      else:\n",
        "        neighbors_of_nearest_node = [n for n in G.neighbors(nearest_node)]\n",
        "        for node_ in neighbors_of_nearest_node:\n",
        "          if marked_node[node_] == 1 and marked_again[node_] == 1: continue\n",
        "          else: queue.append(node_)\n",
        "\n",
        "    return generated_qa_pairs"
      ],
      "metadata": {
        "id": "PP501hSkfvDR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "125c3319-104f-4573-c255-a65b4226b50e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<>:154: SyntaxWarning: \"is not\" with a literal. Did you mean \"!=\"?\n",
            "<>:154: SyntaxWarning: \"is not\" with a literal. Did you mean \"!=\"?\n",
            "<ipython-input-4-7d37bbd4c635>:154: SyntaxWarning: \"is not\" with a literal. Did you mean \"!=\"?\n",
            "  KEs.extend([ke.strip() for ke in KE_sent.split(' SEP>') if ke.strip() is not \"\"])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "conversation_generator = Generate_Conversation_from_Graph()"
      ],
      "metadata": {
        "id": "RvKhXxPkIcKx",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 165
        },
        "outputId": "676035db-d71a-4d55-d286-f1c9e8a7c698"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-7-f5d5fb7e08b5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mconversation_generator\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mGenerate_Conversation_from_Graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'Generate_Conversation_from_Graph' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "context = \"\"\"\n",
        "  One day Mary took a walk to the park.\n",
        "  The park was very close to her house.\n",
        "  One her way to the park she passed her friend Kim's house.\n",
        "  Mary stopped by and asked if Kim wanted to play.\n",
        "  Kim said yes.\n",
        "  Mary and Kim walked together to the park.\n",
        "  John's house was three houses down.\n",
        "  Mary and Kim stopped by to ask John if he wanted to play at the park.\n",
        "  John said no.\n",
        "  He was afraid of being chased by a squirrel.\n",
        "  Mary worried that John didn't like her, but John thought she was a good friend.\n",
        "  So Mary and Kim went to the park to play.\n",
        "  They loved the park.\n",
        "  They loved the flowers, and the swings!\n",
        "  Soon it was dinnertime and the girls went home.\n",
        "  What a lovely day at the park.\n",
        "\"\"\""
      ],
      "metadata": {
        "id": "3NVKLf6JJtB2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "graph_triples = [('the park 1', 'The park 2', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('The park 2', 'the park 3', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('The park 2', 'her 2', {'label': 'Same Sentence', 'color': 'green'}),\n",
        " ('the park 3', 'the park 6', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('the park 3', 'she 3', {'label': 'Same Sentence', 'color': 'green'}),\n",
        " ('the park 6', 'the park 8', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('the park 8', 'the park 12', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('the park 12', 'the park 13', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('the park 13', 'the park 15', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('Mary 1', 'her 2', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('Mary 1', 'the park 1', {'label': 'Same Sentence', 'color': 'green'}),\n",
        " ('her 2', 'her 3', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('her 3', 'she 3', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('her 3', 'Mary 4', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('her 3', 'the park 3', {'label': 'Same Sentence', 'color': 'green'}),\n",
        " ('she 3', 'her 3', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('she 3', 'Kim 3', {'label': 'Same Sentence', 'color': 'green'}),\n",
        " ('Mary 4', 'Mary 6', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('Mary 4', 'Kim 4', {'label': 'Same Sentence', 'color': 'green'}),\n",
        " ('Mary 6', 'Mary 11', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('Mary 6', 'Mary and Kim 6', {'label': 'Same Sentence', 'color': 'green'}),\n",
        " ('Mary 11', 'her 11', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('Mary 11', 'John 11', {'label': 'Same Sentence', 'color': 'green'}),\n",
        " ('her 11', 'she 11', {'label': 'Same Sentence', 'color': 'green'}),\n",
        " ('she 11', 'Mary 12', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('Mary 12', 'Mary and Kim 12', {'label': 'Same Sentence', 'color': 'green'}),\n",
        " (\"her friend Kim 's 3\", 'Kim 4', {'label': 'Coreference', 'color': 'red'}),\n",
        " (\"her friend Kim 's 3\",\n",
        "  'One 3',\n",
        "  {'label': 'Same Sentence', 'color': 'green'}),\n",
        " ('Kim 4', 'Kim 5', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('Kim 5', 'Kim 6', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('Kim 6', 'Kim 12', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('Kim 6', 'the park 6', {'label': 'Same Sentence', 'color': 'green'}),\n",
        " ('Kim 12', 'the park 12', {'label': 'Same Sentence', 'color': 'green'}),\n",
        " ('Mary and Kim 6',\n",
        "  'Mary and Kim 8',\n",
        "  {'label': 'Coreference', 'color': 'red'}),\n",
        " ('Mary and Kim 6', 'Kim 6', {'label': 'Same Sentence', 'color': 'green'}),\n",
        " ('Mary and Kim 8',\n",
        "  'Mary and Kim 12',\n",
        "  {'label': 'Coreference', 'color': 'red'}),\n",
        " ('Mary and Kim 8', 'Mary 8', {'label': 'Same Sentence', 'color': 'green'}),\n",
        " ('Mary and Kim 12', 'They 13', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('Mary and Kim 12', 'Kim 12', {'label': 'Same Sentence', 'color': 'green'}),\n",
        " ('They 13', 'They 14', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('They 13', 'the park 13', {'label': 'Same Sentence', 'color': 'green'}),\n",
        " ('They 14', 'the girls 14', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('the girls 14', 'They 14', {'label': 'Same Sentence', 'color': 'green'}),\n",
        " (\"John 's 7\", 'John 8', {'label': 'Coreference', 'color': 'red'}),\n",
        " (\"John 's 7\", 'John 7', {'label': 'Same Sentence', 'color': 'green'}),\n",
        " ('John 8', 'he 8', {'label': 'Same Sentence', 'color': 'green'}),\n",
        " ('he 8', 'John 9', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('he 8', 'the park 8', {'label': 'Same Sentence', 'color': 'green'}),\n",
        " ('John 9', 'He 10', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('He 10', 'John 11', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('John 11', 'John 11', {'label': 'Coreference', 'color': 'red'}),\n",
        " ('John 11', 'her 11', {'label': 'Same Sentence', 'color': 'green'}),\n",
        " ('One 3', 'her 3', {'label': 'Same Sentence', 'color': 'green'}),\n",
        " ('John 7', 'three 7', {'label': 'Same Sentence', 'color': 'green'}),\n",
        " ('Mary 8', 'Kim 8', {'label': 'Same Sentence', 'color': 'green'}),\n",
        " ('Kim 8', 'John 8', {'label': 'Same Sentence', 'color': 'green'})]"
      ],
      "metadata": {
        "id": "xhCHFEfAJucC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "conversation_generator.generate_full_conversation(context, graph_triples, \"0\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kDultZZQJmNw",
        "outputId": "ea213fab-d75a-4861-b723-dbf528820584"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Done graph initialization!\n",
            "['What happened one day?', 'What happened one day?', 'What did Mary do one day?']\n",
            "['Mary took a walk to the park.', 'Mary took a walk to the park.', 'took a walk to the park']\n",
            "['And what did she do?', 'What did she do?', 'What did she do?']\n",
            "['asked if Kim wanted to play', 'stopped by', 'She stopped by and asked if Kim wanted']\n",
            "['And what?', 'what?', 'What was the destination?']\n",
            "[\"her friend Kim's house.\", 'The park was very close to her house.', 'The park was very close to her house.']\n",
            "['Was it close?', 'Was it close to her house?', 'Was the park near her house?']\n",
            "['Yes', 'yes', 'Yes']\n",
            "['What was close?', 'What was close by?', 'Where?']\n",
            "['The park', 'The park', 'the park was very close to her house.']\n",
            "['Why?', 'Why?', 'Why?']\n",
            "['The park was very close to her house.', 'The park was very close to her house.', 'The park was very close to her house.']\n",
            "['Who did she pass by while walking to the park?', 'Who was her friend?', 'Whose house did she walk past?']\n",
            "['Kim', 'Kim.', \"Kim's\"]\n",
            "[\"Who's house was she walking to?\", 'What did she pass?', 'Who did she pass?']\n",
            "[\"Kim's\", \"Kim's house\", \"Kim's house\"]\n",
            "['What house?', 'Whose?', 'Whose?']\n",
            "[\"John's\", 'her friend', \"Mary's\"]\n",
            "['Who walked to the park?', 'Who walked there together?', 'Who took the walk to the park?']\n",
            "['Mary and Kim', 'Mary and Kim', 'Mary']\n",
            "['Did she go alone?', 'Did they go together?', 'Did she walk alone?']\n",
            "['No', 'Yes.', 'No']\n",
            "['Who did she visit?', 'Who did she walk to?', 'Who was her friend?']\n",
            "['her friend.', 'her friend', \"John's house\"]\n",
            "['What did she do then?', 'What happened when she passed?', 'What happened there?']\n",
            "['asked if Kim wanted to play.', 'Mary stopped by and asked if Kim wanted', 'she passed her friend']\n",
            "['How did she answer?', 'What happened then?', 'What happened then?']\n",
            "['yes', 'Mary and Kim walked together to the park', 'Mary and Kim walked together to the park']\n",
            "['What happened?', 'What did she do?', 'What happened when she did?']\n",
            "['Mary and Kim walked together to the park', 'took a walk to the park', 'Mary took a walk to the park']\n",
            "['Who did she ask?', 'Who did she ask?', 'Who did she ask?']\n",
            "['John', 'John', 'John']\n",
            "['What did they do then?', 'And what happened then?', 'And then what?']\n",
            "['they walked together to the park.', 'Mary and Kim walked together to the park', 'To play.']\n",
            "['Who went to the park?', 'Who went to the park?', 'Who walked together?']\n",
            "['Mary and Kim', 'Mary and Kim', 'Mary and Kim']\n",
            "[\"Who's house?\", 'What home did she pass?', 'Where did she pass?']\n",
            "[\"her friend Kim's\", \"John's house\", 'The park']\n",
            "[\"Who's house was it?\", 'What did she pass?', \"Who's house did she pass?\"]\n",
            "[\"John's\", 'the park', \"her friend Kim's\"]\n",
            "['Where did she see her friend?', 'Who did she pass?', 'Whose house?']\n",
            "['the park', 'her friend Kim', 'John']\n",
            "['Did she do anything else?', 'Who else?', 'Who did she ask?']\n",
            "['Yes', 'John', 'John']\n",
            "['Who did she ask?', 'Who did she ask about the park?', 'Who did she ask if she wanted to play?']\n",
            "['John', 'John', 'John']\n",
            "['Who was he asking about?', 'Who went there?', 'Who walked to the park?']\n",
            "['John', 'Mary and Kim', 'Mary and Kim']\n",
            "['Who went to the park?', 'Who went?', 'Who went to the park to play?']\n",
            "['Mary and Kim', 'Mary and Kim', 'Mary and Kim']\n",
            "['What did they go to?', 'Where did they go?', 'Where did they go?']\n",
            "['the park', 'The park', 'the park']\n",
            "['Who did she go with to the park?', 'Who took her there?', 'Who walked with him?']\n",
            "['Mary and Kim', 'John', 'Mary and Kim']\n",
            "['Where did she see him?', 'Where did she pass?', \"Who's house did she pass?\"]\n",
            "[\"Kim's house\", \"Kim's house\", \"Kim's\"]\n",
            "['Was Kim interested?', 'Did she agree?', 'Did Kim want to go?']\n",
            "['yes', 'no', 'yes']\n",
            "['What did Kim say?', 'How did she respond?', 'What did Kim answer?']\n",
            "['yes', 'No', 'yes']\n",
            "['How did John feel about Mary?', 'Why did she go to the park?', 'Why did she go?']\n",
            "[\"he didn't like her\", 'to play', 'to play']\n",
            "['Why did Mary think John was a good friend?', \"Why did she think he wasn't going?\", 'Why did Mary say yes to John?']\n",
            "['John thought she was a good friend', 'He was afraid of being chased by', 'He was afraid of being chased by']\n",
            "[\"Why didn't John like her?\", 'Why did Mary and Kim go to the park together?', 'Why did John not want to go?']\n",
            "['He was afraid of being chased by', 'to play', 'He was afraid of being chased by']\n",
            "['Who did Mary think was a good friend?', 'Who did Mary think was a good friend?', 'Who was her friend?']\n",
            "[\"Mary worried that John didn't like her\", 'John', 'Kim']\n",
            "['Who went to the park together?', 'Who went there?', 'Who walked together to the park?']\n",
            "['Mary and Kim', 'Mary and Kim', 'Mary and Kim']\n",
            "['Did they go to the park together?', 'Did they go by themselves?', 'Did they go?']\n",
            "['yes', 'no', 'yes']\n",
            "['Who went to the park together?', 'Who went to play?', 'Who went?']\n",
            "['Mary and Kim', 'Mary and Kim', 'Mary and Kim']\n",
            "['Who went with Mary to the park?', 'Who went?', 'Who went with her to the park?']\n",
            "['Kim', 'Mary and Kim', 'Mary and Kim']\n",
            "['What did they go to the park for?', 'What did they do there?', 'What did she do at the park?']\n",
            "['to play', 'played.', 'played']\n",
            "['To play and John?', 'What did they go there?', 'So what did they go there?']\n",
            "['no', 'to play', 'to play']\n",
            "['Why did she go?', 'Why did they go there?', 'What did they do?']\n",
            "['to play.', 'to play', 'walked together']\n",
            "['Who went to the park?', 'Who went to the park to play?', 'Who went with her to the park?']\n",
            "['Mary and Kim', 'Mary and Kim', 'Mary and Kim']\n",
            "['What happened when she came to the park?', 'What did she do next?', 'What happened next?']\n",
            "['Mary and Kim walked together', 'Mary took a walk to the park', 'they went home']\n",
            "['Who did Mary ask?', 'Who did she ask about it to?', 'Who did they ask about the park?']\n",
            "['John', 'Kim', 'John']\n",
            "['Who went to the park?', 'Who went to the park together?', 'Who went to play?']\n",
            "['Mary and Kim', 'Mary and Kim', 'Mary and Kim']\n",
            "['What did they go to the park for?', 'What did they do at the park?', 'What did they go there for?']\n",
            "['to play', 'played', 'to play']\n",
            "['Where?', 'Where?', 'where?']\n",
            "['The park was very close to her house', 'garden', 'the park']\n",
            "['With who?', 'Who did?', 'Who did?']\n",
            "['Mary and Kim', 'Mary and Kim', 'Mary and Kim']\n",
            "['What did Mary think about John?', 'Why did they go?', 'Why?']\n",
            "['she thought he was a good friend', 'they loved the park', 'they loved the park']\n",
            "['And what else?', 'How about John?', 'What about John?']\n",
            "['they loved the park', 'no', 'no']\n",
            "['And what about John?', 'What about John?', 'And John?']\n",
            "[\"he didn't like her\", 'no', 'no']\n",
            "['What did she do instead?', 'What did she do next?', 'How did they find out about him?']\n",
            "['went to the park', 'went to the park', 'by walking']\n",
            "['Who did she ask for the park to play with?', 'Who did she play with?', 'Who did Mary and Kim stop by her friend to play at the park?']\n",
            "['John', 'John', 'John']\n",
            "['Who did she ask about him?', 'Who went to the park?', 'Who went?']\n",
            "['Kim', 'Mary and Kim', 'Mary and Kim']\n",
            "['Did they like it?', 'Did they love it?', 'Did they enjoy the park?']\n",
            "['yes', 'yes', 'yes']\n",
            "['What happened next?', 'Did anyone answer?', 'What did she do?']\n",
            "['the girls went home', 'no', 'went to the park to play']\n",
            "['Did he agree to play there?', 'Did John want to go to the park?', 'Did John want to go?']\n",
            "['no', 'no', 'no']\n",
            "['Where did Mary and Kim play?', 'Where did they go?', 'Where did they go to play?']\n",
            "['the park', 'the park', 'to the park']\n",
            "['Why did they go to the park?', 'Why did they go to the park?', 'Why did she and Kim go to the park?']\n",
            "['to play', 'to play', 'to play.']\n",
            "['Who went to the park?', 'Who went to play?', 'Who went to play?']\n",
            "['Mary and Kim', 'Mary and Kim', 'Mary and Kim']\n",
            "['What was John afraid of?', 'What was he afraid of?', 'What did he fear to be?']\n",
            "['being chased by a squirrel.', 'being chased by a squirrel', 'chased by a squirrel']\n",
            "['What did they do?', 'What did they do next?', 'What did they go to the park for?']\n",
            "['they loved the park', 'they went home.', 'to play']\n",
            "['Where did they go?', 'Where did she go?', 'Where did they go?']\n",
            "['the park', 'the park', 'the park']\n",
            "['Who went to play?', 'Who went to the park for fun?', 'Who went to the park to play?']\n",
            "['Mary and Kim', 'Mary and Kim', 'Mary and Kim']\n",
            "['What happened to the park first?', 'What did she do at the park then?', 'Why did she go to the park?']\n",
            "['it was close to her house.', 'went to the park to play.', 'to play']\n",
            "['Who did Mary ask?', 'Who did they play with there?', 'Who did she ask?']\n",
            "['John', 'Kim', 'John']\n",
            "['Who did she see at the park?', 'Who went to the park?', 'Who went to play?']\n",
            "['Kim', 'Mary and Kim', 'Mary and Kim']\n",
            "['What was it that they did after they played?', 'When did the girls go home?', 'What happened when they were gone?']\n",
            "['they loved the park', 'soon it was dinnertime', 'they went home']\n",
            "['What time did they get home?', 'What time did they go home?', 'What time did they get home?']\n",
            "['dinnertime', 'dinnertime', 'dinnertime']\n",
            "['Did they like the park?', 'Did they enjoy the park?', 'Did they like the park?']\n",
            "['yes', 'yes', 'yes']\n",
            "[\"Was John's home nearby?\", 'Did the park have an easy walk?', \"Was John's house near Mary's?\"]\n",
            "['yes', 'yes', 'yes']\n",
            "[\"How many houses down was John's?\", 'How many houses were down when John came?', \"How far from the park was John's house?\"]\n",
            "['three', 'three', 'three houses down']\n",
            "['How many?', 'How many did he live?', \"How many down were Mary and Kim's?\"]\n",
            "['three', 'three', 'three']\n",
            "['What did Mary and Kim do next?', 'What did Mary and Kim do there?', 'What did Mary and Kim do then?']\n",
            "['went to the park', 'asked John if he wanted to play', 'went to the park']\n",
            "['Who wanted to go play at the park?', 'Who stopped by to play?', 'Who decided not to play at the park?']\n",
            "['Mary and Kim', 'Mary and Kim', 'John']\n",
            "['Did the girls like the park?', 'Why did the girls go to the park?', 'What did they enjoy the park?']\n",
            "['yes', 'to play', 'the flowers, and the swings']\n",
            "['How many more down?', 'Did he live there?', 'Did Mary want him to play?']\n",
            "['four', 'no', 'no']\n",
            "['How many were up?', 'How many houses down?', 'What did that mean?']\n",
            "['five', 'three', 'one house']\n",
            "['What did they enjoy in the park?', 'Why did they go to the park?', 'How did the girls feel about the park?']\n",
            "['the flowers, and the swings', 'to play', 'They loved it']\n",
            "['Did his house have that much space?', 'Did it have a lot of land around it?', 'Was there more than one house down?']\n",
            "['no', 'Yes', 'yes']\n",
            "['How many were there?', \"How far was it from Mary's?\", 'And how many houses down was his house?']\n",
            "['Three', 'a walk', 'three']\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['',\n",
              " 'What happened one day?',\n",
              " 'Mary took a walk to the park.',\n",
              " 'Why?',\n",
              " 'The park was very close to her house.',\n",
              " 'What did she pass?',\n",
              " \"Kim's house\",\n",
              " 'What happened when she passed?',\n",
              " 'Mary stopped by and asked if Kim wanted',\n",
              " 'Who did she ask about the park?',\n",
              " 'John',\n",
              " 'What did Kim say?',\n",
              " 'yes',\n",
              " 'What did they do at the park?',\n",
              " 'played',\n",
              " 'What did Mary think about John?',\n",
              " 'she thought he was a good friend',\n",
              " 'What time did they get home?',\n",
              " 'dinnertime',\n",
              " \"How many houses down was John's?\",\n",
              " 'three']"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "4LyQKQY1KfDI"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}