Datasci_266_summarization_incontextlearning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Install libraries"
      ],
      "metadata": {
        "id": "ljMsacYlNZJ8"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ufHWDmcDiqjk"
      },
      "outputs": [],
      "source": [
        "!pip install -q sentencepiece\n",
        "!pip -q install git+https://github.com/huggingface/transformers\n",
        "!pip install -q evaluate\n",
        "!pip install -q datasets\n",
        "!pip install -q pandas\n",
        "!pip install -q rouge_score\n",
        "!pip install -q nltk\n",
        "!pip install bert_score\n",
        "!pip install sentence-transformers\n",
        "!pip install google-generativeai\n",
        "!pip install -U bitsandbytes\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Create models and tokenizers"
      ],
      "metadata": {
        "id": "gfYA4HdgNxDT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer , AutoModelForCausalLM, BitsAndBytesConfig\n",
        "from transformers import T5Tokenizer, TFT5ForConditionalGeneration\n",
        "import torch\n",
        "from torchsummary import summary\n",
        "from google.colab import userdata\n",
        "\n",
        "def del_model(model, tokenizer):\n",
        "    del model\n",
        "    del tokenizer\n",
        "    torch.cuda.empty_cache()\n",
        "\n",
        "def create_t5_model(model_size):\n",
        "    tokenizer = T5Tokenizer.from_pretrained(model_size)\n",
        "    model = TFT5ForConditionalGeneration.from_pretrained(model_size)\n",
        "    return model, tokenizer\n",
        "\n",
        "def create_mistral_model(model_size):\n",
        "    quantization_config = BitsAndBytesConfig(load_in_4bit=True,\n",
        "                                         llm_int4_enable_fp32_cpu_offload=True)\n",
        "\n",
        "    model = AutoModelForCausalLM.from_pretrained(\"mistralai/Mistral-7B-Instruct-v0.1\",\n",
        "                        torch_dtype=torch.float32,\n",
        "                        device_map='auto',\n",
        "                        quantization_config=quantization_config,\n",
        "                        use_auth_token=\"userdata.get('MISTRAL')\n",
        "                        )\n",
        "    tokenizer = AutoTokenizer.from_pretrained(\"mistralai/Mistral-7B-Instruct-v0.1\",\n",
        "                        use_auth_token=userdata.get('MISTRAL')\n",
        "                        )\n",
        "    return model, tokenizer\n"
      ],
      "metadata": {
        "id": "KKaOXa8ji_77"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Import utility library"
      ],
      "metadata": {
        "id": "Myoa--heQlKg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#!pip install summary_utils\n",
        "!wget https://raw.githubusercontent.com/sleighton2022/datasci266-final-project/main/summary_utils.py\n",
        "from summary_utils import SummaryEvaluator, DatasetManager, SummaryModel"
      ],
      "metadata": {
        "id": "i-epuQrFjNgC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Experiments"
      ],
      "metadata": {
        "id": "em0PH2EfQs-8"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Key parameters"
      ],
      "metadata": {
        "id": "f25MbEOhNpmG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# %% tags=[\"parameters\"]\n",
        "seed = 42\n",
        "sample_size = 2\n",
        "model_type = \"t5\"\n",
        "#model_type = \"mistral\"\n",
        "model_size = \"t5-base\"\n",
        "prompt_template = \"tldr: {document}\""
      ],
      "metadata": {
        "id": "3P9ZROi2j57I"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Initialization"
      ],
      "metadata": {
        "id": "rraRjy_2UgvC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "results = {}\n",
        "prompt_template_set = {}"
      ],
      "metadata": {
        "id": "IuvPK1W0VA6n"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Load the dataset"
      ],
      "metadata": {
        "id": "rDdT9CC8Qz2o"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Load dataset\n",
        "dataset_manager = DatasetManager(seed=seed,sample_size=sample_size)\n",
        "sampled_dataset = dataset_manager.load_sampled_dataset(dataset_label=\"test\")"
      ],
      "metadata": {
        "id": "lGPSoAJPjRqA"
      },
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "l46IYQApUutV"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Helper functions"
      ],
      "metadata": {
        "id": "UkYZz-w2lTbQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate_prompts(model,tokenizer,model_size,prompt_template_set=prompt_template_set,eval_summarizer=None):\n",
        "  summary_model = SummaryModel(model, tokenizer)\n",
        "  summary_evaluator = SummaryEvaluator()\n",
        "  all_generated_summaries = []\n",
        "  \"\"\"\n",
        "  if eval_summarizer is not None:\n",
        "    gen_summarizer = eval_summarizer\n",
        "  else:\n",
        "    gen_summarizer = summary_model.default_summarizer\n",
        "  \"\"\"\n",
        "  for prompt_name, prompt_template in prompt_template_set.items():\n",
        "    print(prompt_name, prompt_template)\n",
        "    results_name = model_size + \"-\" + prompt_name\n",
        "    generated_summaries = summary_model.generate_summaries(sampled_dataset,prompt_template=prompt_template, gen_summarizer=eval_summarizer)\n",
        "    results[results_name] = summary_evaluator.evaluate(sampled_dataset, generated_summaries)\n",
        "    all_generated_summaries.append(generated_summaries)\n",
        "  del_model(model, tokenizer)\n",
        "  return all_generated_summaries"
      ],
      "metadata": {
        "id": "uoz95kqqubAw"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def mistral_summarizer(self,prompt):\n",
        "# Tokenize and generate\n",
        "        input_ids = self.tokenizer(prompt, return_tensors=\"pt\").input_ids\n",
        "        if torch.cuda.is_available():\n",
        "            input_ids = input_ids.cuda()\n",
        "        output_ids = self.model.generate(\n",
        "            input_ids,\n",
        "            max_new_tokens=50,\n",
        "            num_beams=4,\n",
        "            early_stopping=True,\n",
        "            pad_token_id=self.tokenizer.eos_token_id  # Set EOS token for stopping\n",
        "        )\n",
        "\n",
        "        # Decode and return ONLY the summary portion\n",
        "        output_text = self.tokenizer.decode(output_ids[0], skip_special_tokens=True)\n",
        "\n",
        "        # Find the index where the summary starts (after the prompt)\n",
        "        summary_start = output_text.find(prompt) + len(prompt) + 1  # +1 to skip the newline\n",
        "\n",
        "        # Return only the text after the prompt\n",
        "        return output_text[summary_start:].strip()  # Strip any extra whitespace\n"
      ],
      "metadata": {
        "id": "oWTCoNkJPOjZ"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Initialize an empty list to hold the data\n",
        "def create_dataframe(results):\n",
        "  rows = []\n",
        "\n",
        "  # Iterate over the data and flatten the structure\n",
        "  for prompt, metrics in results.items():\n",
        "    row = {'prompt': prompt}\n",
        "    row.update({\n",
        "        'rouge1': metrics['rouge']['rouge1'],\n",
        "        'rouge2': metrics['rouge']['rouge2'],\n",
        "        'rougeL': metrics['rouge']['rougeL'],\n",
        "        'bleu': metrics['bleu'],\n",
        "        'bertscore': metrics['bertscore'],\n",
        "        'vector_similarity': metrics['vector_similarity']\n",
        "    })\n",
        "    rows.append(row)\n",
        "\n",
        "  # Create a DataFrame from the list of rows\n",
        "  df = pd.DataFrame(rows)\n",
        "\n",
        "  # Display the DataFrame\n",
        "  print(df)"
      ],
      "metadata": {
        "id": "YIbkqYV40fan"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def print_summaries(summaries):\n",
        "  for summary in summaries:\n",
        "    print(summary)"
      ],
      "metadata": {
        "id": "KYsZ_f_-puPD"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 1 Prompts"
      ],
      "metadata": {
        "id": "m_zokozxRBUS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "prompt_template_set = {}\n",
        "prompt_template_set[\"prompt1\"] = \"Summarize this article: {document}\"\n",
        "prompt_template_set[\"prompt2\"] = \"What are the key points of this article: {document}\"\n",
        "prompt_template_set[\"prompt3\"] = \"Summarize this article for a 5th grader: {document}\"\n",
        "prompt_template_set[\"prompt4\"] = \"Write a summary of this article in 50 words: {document}\"\n",
        "prompt_template_set[\"prompt5\"] = \"Summarize the article in 3 bullet points: {document}\""
      ],
      "metadata": {
        "id": "zh1LQRI8hT4F"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### 1.1 T5 base  \n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "aC2EP5JLR03f"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model_size = \"t5-base\"\n",
        "model, tokenizer = create_t5_model(model_size)\n",
        "summaries = evaluate_prompts(model=model,tokenizer=tokenizer,model_size=model_size)"
      ],
      "metadata": {
        "id": "k5J1WqrXi2eD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for summary in summaries:\n",
        "  print(summary)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CrJPt2ahlfIH",
        "outputId": "555b8661-6f85-4fc0-b6d0-215c39da6155"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['21 women were heading to Liverpool when their minibus was hit by a lorry . their friend, 18-year-old Bethany Jones, was killed in the crash . minibus driver James Johnson was jailed for more than six years for causing her death .']\n",
            "[\"21 women were heading to Liverpool when their minibus was hit by a lorry . their friend, 18-year-old Bethany Jones, was killed in the crash . minibus driver James Johnson was jailed for more than six years for causing her friend's death .\"]\n",
            "[\"21 women were heading to Liverpool when their minibus was hit by a lorry . their friend, 18-year-old Bethany Jones, was killed in the crash . minibus driver James Johnson was jailed for more than six years for causing her friend's death .\"]\n",
            "['21 women were heading to Liverpool when their minibus was hit by a lorry . their friend, 18-year-old Bethany Jones, was killed in the crash . minibus driver james johnson was jailed for more than six years in 2013 .']\n",
            "[\"21 women were heading to Liverpool when their minibus was hit by a lorry . their friend, 18-year-old Bethany Jones, was killed in the crash . minibus driver James Johnson was jailed for more than six years for causing her friend's death .\"]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "create_dataframe(results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C3C8hovUoi1H",
        "outputId": "7b800004-bcf9-487a-8979-f34f8f089e4c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "            prompt    rouge1    rouge2    rougeL      bleu  bertscore  \\\n",
            "0  t5-base-prompt1  0.190947  0.024673  0.129306  0.015440   0.501195   \n",
            "1  t5-base-prompt2  0.200283  0.027235  0.131034  0.016077   0.506631   \n",
            "2  t5-base-prompt3  0.194695  0.026244  0.132029  0.014832   0.505272   \n",
            "3  t5-base-prompt4  0.199000  0.024999  0.133052  0.014553   0.503812   \n",
            "4  t5-base-prompt5  0.185086  0.023984  0.126145  0.015732   0.499326   \n",
            "\n",
            "   vector_similarity  \n",
            "0           0.513904  \n",
            "1           0.487193  \n",
            "2           0.533619  \n",
            "3           0.507103  \n",
            "4           0.503888  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### 1.2 T5 small"
      ],
      "metadata": {
        "id": "xd-SlgzDtKNP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model_size = \"t5-small\"\n",
        "model, tokenizer = create_t5_model(model_size)\n",
        "summaries = evaluate_prompts(model=model,tokenizer=tokenizer,model_size=model_size)"
      ],
      "metadata": {
        "id": "GFjOFx6vu4uF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print_summaries(summaries)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x6NYoclNqMTk",
        "outputId": "95eb7230-19d9-4f42-d9ec-a63af9876e48"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['her friend Bethany Jones, 18, was killed while her minibus was hit by a lorry . the driver was jailed for more than six years for causing the crash . she said the help she received from a charity led her to want to support others .']\n",
            "['her friend Bethany Jones, 18, was killed while her minibus was hit by a lorry . the driver was jailed for more than six years for causing the crash . she said the help she received from a charity led her to want to support others .']\n",
            "['her friend Bethany Jones, 18, was killed while her minibus was hit by a lorry on the M62 . she broke her shoulder, back and pelvis and said the help she received from a charity led her to want to support others . the minibus driver was jailed for more than six years for causing the death .']\n",
            "['her friend Bethany Jones, 18, was killed while her minibus was hit by a lorry on the M62 . she broke her shoulder, back and pelvis and said the help she received from a charity led her to want to support others . the crash made her realise how lucky she had been .']\n",
            "['her friend Bethany Jones, 18, was killed while her friend was badly hurt . the minibus driver was jailed for more than six years for causing the crash . she said the help she received from a charity led her to want to support others .']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "create_dataframe(results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ouNxqd9ntiIw",
        "outputId": "de919cd9-d275-4804-d3d1-5d79440f5e35"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "             prompt    rouge1    rouge2    rougeL      bleu  bertscore  \\\n",
            "0   t5-base-prompt1  0.190947  0.024673  0.129306  0.015440   0.501195   \n",
            "1   t5-base-prompt2  0.200283  0.027235  0.131034  0.016077   0.506631   \n",
            "2   t5-base-prompt3  0.194695  0.026244  0.132029  0.014832   0.505272   \n",
            "3   t5-base-prompt4  0.199000  0.024999  0.133052  0.014553   0.503812   \n",
            "4   t5-base-prompt5  0.185086  0.023984  0.126145  0.015732   0.499326   \n",
            "5  t5-small-prompt1  0.191549  0.021002  0.128870  0.013668   0.496701   \n",
            "6  t5-small-prompt2  0.187117  0.020717  0.128254  0.013492   0.493760   \n",
            "7  t5-small-prompt3  0.188544  0.022267  0.130077  0.013937   0.491193   \n",
            "8  t5-small-prompt4  0.178968  0.017083  0.121125  0.012290   0.495284   \n",
            "9  t5-small-prompt5  0.181769  0.025169  0.130003  0.014895   0.493805   \n",
            "\n",
            "   vector_similarity  \n",
            "0           0.513904  \n",
            "1           0.487193  \n",
            "2           0.533619  \n",
            "3           0.507103  \n",
            "4           0.503888  \n",
            "5           0.426731  \n",
            "6           0.436847  \n",
            "7           0.422234  \n",
            "8           0.448570  \n",
            "9           0.432336  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### 1.3 T5 large  "
      ],
      "metadata": {
        "id": "4Ok0Loz0tVDW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model_size = \"t5-large\"\n",
        "model, tokenizer = create_t5_model(model_size)\n",
        "summaries = evaluate_prompts(model=model,tokenizer=tokenizer,model_size=model_size)"
      ],
      "metadata": {
        "id": "e0YkCuWZu_2C"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print_summaries(summaries)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y5K_29wwrEQo",
        "outputId": "06943acd-ad58-44fb-b38b-0265fe21db8b"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['minibus driver jailed for more than six years for causing death of friend . Sarah Johnson broke her shoulder, back and pelvis in the crash . Ms Johnson said crash made her realise how lucky she was .']\n",
            "['minibus driver jailed for more than six years for causing death of friend . Sarah Johnson broke her shoulder, back and pelvis in minibus crash . Ms Johnson said crash made her realise how lucky she was .']\n",
            "['minibus driver jailed for more than six years for causing death of friend . Sarah Johnson broke her shoulder, back and pelvis in the crash . Ms Johnson says crash made her realise how lucky she was .']\n",
            "['minibus driver james johnson jailed for more than six years for causing hen party crash . friend Bethany Jones, 18, was killed in the crash on the M62 in london . minibus passenger Sarah Johnson broke her shoulder, back and pelvis . she said the crash had made her realise how lucky she was .']\n",
            "['minibus driver jailed for more than six years for causing death of friend . Sarah Johnson broke her shoulder, back and pelvis in the crash . Ms Johnson said crash made her realise how lucky she was .']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "create_dataframe(results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "B0LKyCT_typE",
        "outputId": "7f03616d-0e75-4f5a-dc33-5a8e2c61d294"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              prompt    rouge1    rouge2    rougeL      bleu  bertscore  \\\n",
            "0    t5-base-prompt1  0.190947  0.024673  0.129306  0.015440   0.501195   \n",
            "1    t5-base-prompt2  0.200283  0.027235  0.131034  0.016077   0.506631   \n",
            "2    t5-base-prompt3  0.194695  0.026244  0.132029  0.014832   0.505272   \n",
            "3    t5-base-prompt4  0.199000  0.024999  0.133052  0.014553   0.503812   \n",
            "4    t5-base-prompt5  0.185086  0.023984  0.126145  0.015732   0.499326   \n",
            "5   t5-small-prompt1  0.191549  0.021002  0.128870  0.013668   0.496701   \n",
            "6   t5-small-prompt2  0.187117  0.020717  0.128254  0.013492   0.493760   \n",
            "7   t5-small-prompt3  0.188544  0.022267  0.130077  0.013937   0.491193   \n",
            "8   t5-small-prompt4  0.178968  0.017083  0.121125  0.012290   0.495284   \n",
            "9   t5-small-prompt5  0.181769  0.025169  0.130003  0.014895   0.493805   \n",
            "10  t5-large-prompt1  0.204444  0.029873  0.142275  0.016052   0.510986   \n",
            "11  t5-large-prompt2  0.202771  0.027891  0.140294  0.015622   0.505149   \n",
            "12  t5-large-prompt3  0.195561  0.026718  0.133206  0.014825   0.506940   \n",
            "13  t5-large-prompt4  0.189132  0.027807  0.133507  0.015280   0.503062   \n",
            "14  t5-large-prompt5  0.196144  0.029547  0.130790  0.015234   0.503467   \n",
            "\n",
            "    vector_similarity  \n",
            "0            0.513904  \n",
            "1            0.487193  \n",
            "2            0.533619  \n",
            "3            0.507103  \n",
            "4            0.503888  \n",
            "5            0.426731  \n",
            "6            0.436847  \n",
            "7            0.422234  \n",
            "8            0.448570  \n",
            "9            0.432336  \n",
            "10           0.494955  \n",
            "11           0.470514  \n",
            "12           0.499282  \n",
            "13           0.479829  \n",
            "14           0.486888  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### 1.5 Mistral"
      ],
      "metadata": {
        "id": "eoR7sKw59Q3Y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#Code to load mistral model\n",
        "model_size = \"mistral\"\n",
        "model, tokenizer = create_mistral_model(model_size)\n",
        "summaries = evaluate_prompts(model=model,tokenizer=tokenizer,model_size=model_size,eval_summarizer=mistral_summarizer)"
      ],
      "metadata": {
        "id": "Kj1obIMLEV6O"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print_summaries(summaries)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZcR1mLty-mIZ",
        "outputId": "bd5c1871-7483-4bee-a5fe-45693cea8376"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['Sarah Johnson was one of 21 women on a minibus that was hit by a lorry on the M62 in 2013. Her friend Bethany Jones was killed in the crash, and Sarah']\n",
            "['Ms Johnson is now working as a support worker for Day One, helping other victims of major trauma.']\n",
            "['Sarah Johnson was in a minibus with her friends when it was hit by a lorry on the M62. Her friend Bethany Jones was killed and Sarah was badly hurt. The minibus driver, James Johnson']\n",
            "['Sarah Johnson was one of 21 women on a minibus that was hit by a lorry on the M62. Her friend Bethany Jones, 18, was killed and several others were badly hurt.']\n",
            "['Ms Johnson is now working as a support worker for Day One, helping others who have been through similar experiences.']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "create_dataframe(results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GhVQSVyI6Die",
        "outputId": "1f1610b7-890b-4346-8e87-477b176a24c3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "             prompt    rouge1    rouge2    rougeL      bleu  bertscore  \\\n",
            "0  t5-large-prompt1  0.079800  0.020845  0.060078  0.005461   0.470786   \n",
            "1  t5-large-prompt2  0.075691  0.017116  0.054823  0.004563   0.469554   \n",
            "2  t5-large-prompt3  0.081605  0.022617  0.060730  0.006416   0.475210   \n",
            "3  t5-large-prompt4  0.101566  0.024055  0.069222  0.006478   0.483945   \n",
            "4  t5-large-prompt5  0.089638  0.019487  0.061460  0.005540   0.470911   \n",
            "\n",
            "   vector_similarity  \n",
            "0           0.599967  \n",
            "1           0.596550  \n",
            "2           0.581367  \n",
            "3           0.555626  \n",
            "4           0.586925  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "ev8CiLP29Qy-"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 2 Templates"
      ],
      "metadata": {
        "id": "JatL8nOh4KUe"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "prompt_template_set = {}\n",
        "\n",
        "prompt_template_set[\"template1\"] = \"\"\"\n",
        "Input:\n",
        "Article: {document}\n",
        "\n",
        "Task: Summarize the above article.\n",
        "\n",
        "Output:\n",
        "Summary:\n",
        "\"\"\"\n",
        "\n",
        "prompt_template_set[\"template2\"] = \"\"\"\n",
        "Input:\n",
        "Article: {document}\n",
        "\n",
        "Task: Extract and summarize the key points from the article.\n",
        "\n",
        "Output:\n",
        "Key Points Summary:\n",
        "\"\"\"\n",
        "\n",
        "prompt_template_set[\"template3\"] = \"\"\"Input:\n",
        "Article: {document}\n",
        "\n",
        "Task: Summarize this article for a 5th grader.\n",
        "\n",
        "Output:\n",
        "Summary (50 words):\n",
        "\"\"\"\n",
        "\n",
        "\n",
        "prompt_template_set[\"template4\"] = \"\"\"Input:\n",
        "Article: {document}\n",
        "\n",
        "Task: Summarize the article in approximately 100 words.\n",
        "\n",
        "Output:\n",
        "Summary (50 words):\n",
        "\"\"\"\n",
        "\n",
        "prompt_template_set[\"template5\"] = \"\"\"Input:\n",
        "Article: {document}\n",
        "\n",
        "Task: Summarize the article in bullet points.\n",
        "\n",
        "Output:\n",
        "Summary:\n",
        "- bullet_point_1\n",
        "- bullet_point_2\n",
        "- bullet_point_3\n",
        "\"\"\""
      ],
      "metadata": {
        "id": "Ebzg477cz6RW"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### 2.1 T5 base"
      ],
      "metadata": {
        "id": "ZWfiJoqO9R3v"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model_size = \"t5-base\"\n",
        "model, tokenizer = create_t5_model(model_size)\n",
        "summaries = evaluate_prompts(model=model,tokenizer=tokenizer,model_size=model_size)"
      ],
      "metadata": {
        "id": "Aq9TO82DjpGz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print_summaries(summaries)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9llBvUxqyiVE",
        "outputId": "2cb9691b-c3ae-4504-b404-8c4e4d0c2304"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['21 women were heading to Liverpool when their minibus was hit by a lorry . their friend, 18-year-old Bethany Jones, was killed in the crash . minibus driver James Johnson was jailed for more than six years for causing her death .']\n",
            "[\"21 women were heading to Liverpool when their minibus was hit by a lorry . their friend, 18-year-old Bethany Jones, was killed in the crash . minibus driver James Johnson was jailed for more than six years for causing her friend's death .\"]\n",
            "[\"21 women were heading to Liverpool when their minibus was hit by a lorry . their friend, 18-year-old Bethany Jones, was killed in the crash . minibus driver James Johnson was jailed for more than six years for causing her friend's death .\"]\n",
            "['21 women were heading to Liverpool when their minibus was hit by a lorry . their friend, 18-year-old Bethany Jones, was killed in the crash . minibus driver james johnson was jailed for more than six years in 2013 .']\n",
            "[\"21 women were heading to Liverpool when their minibus was hit by a lorry . their friend, 18-year-old Bethany Jones, was killed in the crash . minibus driver James Johnson was jailed for more than six years for causing her friend's death .\"]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "create_dataframe(results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b8z__V_GHsST",
        "outputId": "2080d568-eded-4243-a912-db76640d9715"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              prompt    rouge1    rouge2    rougeL      bleu  bertscore  \\\n",
            "0  t5-base-template1  0.185725  0.019665  0.126126  0.012149   0.499389   \n",
            "1  t5-base-template2  0.196112  0.026284  0.125525  0.014168   0.496072   \n",
            "2  t5-base-template3  0.187986  0.027379  0.123386  0.014340   0.497435   \n",
            "3  t5-base-template4  0.191913  0.024529  0.124188  0.014073   0.497428   \n",
            "4  t5-base-template5  0.176543  0.025939  0.107458  0.013176   0.486879   \n",
            "\n",
            "   vector_similarity  \n",
            "0           0.492220  \n",
            "1           0.476907  \n",
            "2           0.483379  \n",
            "3           0.493260  \n",
            "4           0.468765  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### 2.2 T5 small"
      ],
      "metadata": {
        "id": "3tGPI_pc9ZNb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model_size = \"t5-small\"\n",
        "model, tokenizer = create_t5_model(model_size)\n",
        "summaries = evaluate_prompts(model=model,tokenizer=tokenizer,model_size=model_size)"
      ],
      "metadata": {
        "id": "unAJd7xaHxo8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print_summaries(summaries)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5opUlHUAzaaW",
        "outputId": "30548380-3132-4e9d-9bb1-56a3a66f9bfe"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['her friend Bethany Jones, 18, was killed while her minibus was hit by a lorry . the driver was jailed for more than six years for causing the crash . she said the help she received from a charity led her to want to support others .']\n",
            "['her friend Bethany Jones, 18, was killed while her minibus was hit by a lorry . the driver was jailed for more than six years for causing the crash . she said the help she received from a charity led her to want to support others .']\n",
            "['her friend Bethany Jones, 18, was killed while her minibus was hit by a lorry on the M62 . she broke her shoulder, back and pelvis and said the help she received from a charity led her to want to support others . the minibus driver was jailed for more than six years for causing the death .']\n",
            "['her friend Bethany Jones, 18, was killed while her minibus was hit by a lorry on the M62 . she broke her shoulder, back and pelvis and said the help she received from a charity led her to want to support others . the crash made her realise how lucky she had been .']\n",
            "['her friend Bethany Jones, 18, was killed while her friend was badly hurt . the minibus driver was jailed for more than six years for causing the crash . she said the help she received from a charity led her to want to support others .']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "create_dataframe(results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pNkxQEGeMv58",
        "outputId": "0425a9d0-1824-49ad-b47c-344c9f420ce6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "               prompt    rouge1    rouge2    rougeL      bleu  bertscore  \\\n",
            "0   t5-base-template1  0.185725  0.019665  0.126126  0.012149   0.499389   \n",
            "1   t5-base-template2  0.196112  0.026284  0.125525  0.014168   0.496072   \n",
            "2   t5-base-template3  0.187986  0.027379  0.123386  0.014340   0.497435   \n",
            "3   t5-base-template4  0.191913  0.024529  0.124188  0.014073   0.497428   \n",
            "4   t5-base-template5  0.176543  0.025939  0.107458  0.013176   0.486879   \n",
            "5  t5-small-template1  0.178138  0.024033  0.125536  0.014100   0.491026   \n",
            "6  t5-small-template2  0.174037  0.021258  0.121758  0.013361   0.495665   \n",
            "7  t5-small-template3  0.167127  0.021007  0.119920  0.013213   0.492251   \n",
            "8  t5-small-template4  0.173759  0.021311  0.117255  0.013748   0.493222   \n",
            "9  t5-small-template5  0.175931  0.020062  0.117802  0.013313   0.491720   \n",
            "\n",
            "   vector_similarity  \n",
            "0           0.492220  \n",
            "1           0.476907  \n",
            "2           0.483379  \n",
            "3           0.493260  \n",
            "4           0.468765  \n",
            "5           0.416400  \n",
            "6           0.438146  \n",
            "7           0.410422  \n",
            "8           0.418657  \n",
            "9           0.415814  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### 2.3 T5 large"
      ],
      "metadata": {
        "id": "DhSkqHh59drX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model_size = \"t5-large\"\n",
        "model, tokenizer = create_t5_model(model_size)\n",
        "summaries = evaluate_prompts(model=model,tokenizer=tokenizer,model_size=model_size)"
      ],
      "metadata": {
        "id": "9TtaU6TZM5Um"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print_summaries(summaries)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JBAZ4V-l12K5",
        "outputId": "0266b73c-954d-4003-8a92-33db73a1ad04"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['minibus driver jailed for more than six years for causing death of friend . Sarah Johnson broke her shoulder, back and pelvis in the crash . Ms Johnson said crash made her realise how lucky she was .']\n",
            "['minibus driver jailed for more than six years for causing death of friend . Sarah Johnson broke her shoulder, back and pelvis in minibus crash . Ms Johnson said crash made her realise how lucky she was .']\n",
            "['minibus driver jailed for more than six years for causing death of friend . Sarah Johnson broke her shoulder, back and pelvis in the crash . Ms Johnson says crash made her realise how lucky she was .']\n",
            "['minibus driver james johnson jailed for more than six years for causing hen party crash . friend Bethany Jones, 18, was killed in the crash on the M62 in london . minibus passenger Sarah Johnson broke her shoulder, back and pelvis . she said the crash had made her realise how lucky she was .']\n",
            "['minibus driver jailed for more than six years for causing death of friend . Sarah Johnson broke her shoulder, back and pelvis in the crash . Ms Johnson said crash made her realise how lucky she was .']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "create_dataframe(results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8yeBr0zGcX7v",
        "outputId": "4860eeb8-9108-4bfb-a565-3816f1cc85de"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                prompt    rouge1    rouge2    rougeL      bleu  bertscore  \\\n",
            "0    t5-base-template1  0.185725  0.019665  0.126126  0.012149   0.499389   \n",
            "1    t5-base-template2  0.196112  0.026284  0.125525  0.014168   0.496072   \n",
            "2    t5-base-template3  0.187986  0.027379  0.123386  0.014340   0.497435   \n",
            "3    t5-base-template4  0.191913  0.024529  0.124188  0.014073   0.497428   \n",
            "4    t5-base-template5  0.176543  0.025939  0.107458  0.013176   0.486879   \n",
            "5   t5-small-template1  0.178138  0.024033  0.125536  0.014100   0.491026   \n",
            "6   t5-small-template2  0.174037  0.021258  0.121758  0.013361   0.495665   \n",
            "7   t5-small-template3  0.167127  0.021007  0.119920  0.013213   0.492251   \n",
            "8   t5-small-template4  0.173759  0.021311  0.117255  0.013748   0.493222   \n",
            "9   t5-small-template5  0.175931  0.020062  0.117802  0.013313   0.491720   \n",
            "10  t5-large-template1  0.204912  0.032149  0.140912  0.015487   0.504403   \n",
            "11  t5-large-template2  0.212757  0.032126  0.142940  0.015773   0.506131   \n",
            "12  t5-large-template3  0.214555  0.034267  0.148931  0.016520   0.503826   \n",
            "13  t5-large-template4  0.209116  0.033327  0.149599  0.016160   0.504132   \n",
            "14  t5-large-template5  0.210897  0.031550  0.141664  0.014807   0.508710   \n",
            "\n",
            "    vector_similarity  \n",
            "0            0.492220  \n",
            "1            0.476907  \n",
            "2            0.483379  \n",
            "3            0.493260  \n",
            "4            0.468765  \n",
            "5            0.416400  \n",
            "6            0.438146  \n",
            "7            0.410422  \n",
            "8            0.418657  \n",
            "9            0.415814  \n",
            "10           0.490799  \n",
            "11           0.488193  \n",
            "12           0.483680  \n",
            "13           0.488698  \n",
            "14           0.493486  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### 2.5 Mistral"
      ],
      "metadata": {
        "id": "DgJMi-ALZiDz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model_size = \"mistral\"\n",
        "model, tokenizer = create_mistral_model(model_size)\n",
        "summaries = evaluate_prompts(model=model,tokenizer=tokenizer,model_size=model_size,eval_summarizer=mistral_summarizer)"
      ],
      "metadata": {
        "id": "1sKmWoNuZeYb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print_summaries(summaries)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l_1KZCoby-9c",
        "outputId": "5391939a-8447-469b-d271-6190f8beef6b"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['Sarah Johnson was one of 21 women on a minibus that was hit by a lorry on the M62 in 2013. Her friend Bethany Jones was killed in the crash, and Sarah']\n",
            "['Ms Johnson is now working as a support worker for Day One, helping other victims of major trauma.']\n",
            "['Sarah Johnson was in a minibus with her friends when it was hit by a lorry on the M62. Her friend Bethany Jones was killed and Sarah was badly hurt. The minibus driver, James Johnson']\n",
            "['Sarah Johnson was one of 21 women on a minibus that was hit by a lorry on the M62. Her friend Bethany Jones, 18, was killed and several others were badly hurt.']\n",
            "['Ms Johnson is now working as a support worker for Day One, helping others who have been through similar experiences.']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "create_dataframe(results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w4xEmzOQoIrm",
        "outputId": "38b71a41-43d2-4320-cbe3-16fa6e5aba66"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              prompt    rouge1    rouge2    rougeL      bleu  bertscore  \\\n",
            "0  mistral-template1  0.077878  0.018334  0.059615  0.004902   0.459197   \n",
            "1  mistral-template2  0.070451  0.016662  0.052368  0.004580   0.446344   \n",
            "2  mistral-template3  0.081330  0.019684  0.059085  0.005002   0.466990   \n",
            "3  mistral-template4  0.077298  0.018825  0.058975  0.005481   0.456700   \n",
            "4  mistral-template5  0.052658  0.011128  0.035191  0.003573   0.399282   \n",
            "\n",
            "   vector_similarity  \n",
            "0           0.526428  \n",
            "1           0.502251  \n",
            "2           0.492006  \n",
            "3           0.515477  \n",
            "4           0.476567  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 3 Few-shot learnings"
      ],
      "metadata": {
        "id": "XwQ7_ha3d4rS"
      }
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "tkwUp8uEfl8F"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "learning_dataset = dataset_manager.load_sampled_dataset(dataset_label=\"train\")\n",
        "\n",
        "prompt_template_set = {}\n",
        "prompt_template_set[\"learning1\"] = \"Document: \" + learning_dataset[\"document\"][0] + \"Summary:\" + \\\n",
        "                                   learning_dataset[\"summary\"][0] + \" Summarize the following {document}\"\n",
        "\n",
        "prompt_template_set[\"learning2\"] = \"Document: \" + learning_dataset[\"document\"][0] + \"Summary:\" + \\\n",
        "                                   learning_dataset[\"summary\"][0] + \\\n",
        "                                   \"Document: \" + learning_dataset[\"document\"][1] + \"Summary:\" + \\\n",
        "                                   learning_dataset[\"summary\"][1] + \" Summarize the following {document}\"\n",
        "\n",
        "\"\"\"\n",
        "prompt_template_set[\"learning3\"] = \"Document: \" + learning_dataset[\"document\"][0] + \"Summary:\" + \\\n",
        "                                   learning_dataset[\"summary\"][0] + \\\n",
        "                                   \"Document: \" + learning_dataset[\"document\"][1] + \"Summary:\" + \\\n",
        "                                   learning_dataset[\"summary\"][1] + \\\n",
        "                                   \"Document: \" + learning_dataset[\"document\"][2] + \"Summary:\" + \\\n",
        "                                   learning_dataset[\"summary\"][2] + \" Summarize the following {document}\"\n",
        "\"\"\""
      ],
      "metadata": {
        "id": "W_ngluf-ekJ2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "0CQzbAWAkK6d"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 3.1 T5-large"
      ],
      "metadata": {
        "id": "wqsdsb06Pxmd"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model_size = \"t5-large\"\n",
        "model, tokenizer = create_t5_model(model_size)\n",
        "summaries = evaluate_prompts(model=model,tokenizer=tokenizer,model_size=model_size)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4I8a0bKz3She",
        "outputId": "14dd0c50-1801-4de9-cf77-5dbc09565e61"
      },
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
            "  warnings.warn(\n",
            "All PyTorch model weights were used when initializing TFT5ForConditionalGeneration.\n",
            "\n",
            "All the weights of TFT5ForConditionalGeneration were initialized from the PyTorch model.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFT5ForConditionalGeneration for predictions without further training.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "learning1 Document: In Wales, councils are responsible for funding and overseeing schools.\n",
            "But in England, Mr Osborne's plan will mean local authorities will cease to have a role in providing education.\n",
            "Academies are directly funded by central government and head teachers have more freedom over admissions and to change the way the school works.\n",
            "It is a significant development in the continued divergence of schools systems on either side of Offa's Dyke.\n",
            "And although the Welsh Government will get extra cash to match the money for English schools to extend the school day, it can spend it on any devolved policy area.\n",
            "Ministers have no plans to follow suit.\n",
            "At the moment, governing bodies are responsible for setting school hours and they need ministerial permission to make significant changes.\n",
            "There are already more than 2,000 secondary academies in England and its extension to all state schools is unlikely to shake the Welsh Government's attachment to what they call a \"community, comprehensive model\" for schools.\n",
            "It rejects claims that freedom given to academies can help drive up standards, and it points to academy-free Scotland as the best performing school system in the UK.\n",
            "Education Minister Huw Lewis said there was \"very little evidence to suggest\" academies have a positive impact in driving up standards and Wales would not be following the model.\n",
            "\"The Tories have wasted hundreds of millions of pounds on academies and free schools and as the Chancellor finalises his budget plans to slash vital services even further, he is committing them to wasting even more on a failing endeavour.\n",
            "\"We have no plans to introduce the chaos and waste of academies and free schools here in Wales.\"\n",
            "None of the main parties in May's Assembly election - including the Welsh Conservatives - have said they want to introduce academies in Wales.\n",
            "Owen Hathway, NUT Cymru's policy officer, called the academy plans for England \"scandalous.\".\n",
            "\"There is no evidence that academies work, no evidence that they raise standards, no evidence that they offer better quality education and no evidence that they are what parents and communities want,\" he said.\n",
            "\"Certainly a commitment to comprehensive education is something we would want, and indeed expect, all parties to hold firm to in their manifestos for the forthcoming Welsh election.\"\n",
            "But the Welsh and English schools systems are still linked by a joint arrangement for teachers' pay and conditions.\n",
            "Academies are not tied to these pay scales so in effect Wednesday's announcement will take all English schools out of the system and raise questions about the viability of an England and Wales pay and conditions structure.\n",
            "There is already growing momentum for the devolution of teachers' pay and conditions.\n",
            "Originally sceptical, the Welsh Labour Government is now broadly in favour.\n",
            "Some teaching unions remain opposed because of concern that Welsh teachers would end up being paid less than those in England.\n",
            "Mr Hathway said teachers were concerned it could lead to regional pay.\n",
            "\"At the same time we do of course recognise that the issue of pay is already becoming a grey area due to the negative changes we see taking place in England,\" he said.\n",
            "But an even bigger difference between the schools landscape on either side of the border, appears to make separate arrangements for pay increasingly likely in future.Summary:As Chancellor George Osborne announced all English state schools will become academies, the Welsh Government continues to reject the model here. Summarize the following {document}\n",
            "Summarized document  0\n",
            "academies are directly funded by central government and head teachers have more freedom over admissions and the way the school works . it is a significant development in the continued divergence of schools systems on either side of the border . the Welsh Government will get extra cash to match the money for english schools to extend the school day, but it can spend it on any devolved policy area .\n",
            "Summarized document  1\n",
            "academies are directly funded by central government and head teachers have more freedom over admissions and the way the school works . it is a significant development in the continued divergence of schools systems on either side of offa's dyke . the Welsh Government will get extra cash to match the money for english schools to extend the school day, but it can spend it on any devolved policy area .\n",
            "Average ROUGE score: {'rouge1': 0.04546703296703297, 'rouge2': 0.0, 'rougeL': 0.03447802197802198}\n",
            "Average BLEU score: 0.005227371699708218\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
            "  warnings.warn(\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
            "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Average BERTSCORE score: 0.4012531638145447\n",
            "Average VECTOR_SIMILARITY score: 0.07358400523662567\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "create_dataframe(results)"
      ],
      "metadata": {
        "id": "fQXrrCj7OozV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print_summaries(summaries)"
      ],
      "metadata": {
        "id": "0PHMcaom3avh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "####3.5 Mistral"
      ],
      "metadata": {
        "id": "KPk842Yd2848"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model_size = \"mistral\"\n",
        "model, tokenizer = create_mistral_model(model_size)\n",
        "summaries = evaluate_prompts(model=model,tokenizer=tokenizer,model_size=model_size,eval_summarizer=mistral_summarizer,prompt_template_set=prompt_template_set)"
      ],
      "metadata": {
        "id": "OZTUspEarZy3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print_summaries(summaries)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cicZjZIc2miv",
        "outputId": "bd72c0a3-adbd-44fd-8b9a-5512b950ff63"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['Ms Johnson, who is now working as a support worker for Day One, said she wanted to give back to the charity that had helped her.\\nShe said: \"It\\'s something I\\'m passionate about.\\n\"It\\'s']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "create_dataframe(results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hqRuaX9ssiiT",
        "outputId": "18003c50-e7ec-43c3-ca23-84ecffa776a3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              prompt    rouge1    rouge2    rougeL      bleu  bertscore  \\\n",
            "0  mistral-learning1  0.024205  0.006380  0.018101  0.001472   0.375454   \n",
            "1  mistral-learning2  0.025150  0.007356  0.018224  0.001685   0.375454   \n",
            "\n",
            "   vector_similarity  \n",
            "0           0.073044  \n",
            "1           0.073044  \n"
          ]
        }
      ]
    }
  ]
}