T5.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "27fb5c67b0fa4b0dac53106f4ec76c45": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_580d0772992f4b23b60dc4c953f5689e",
              "IPY_MODEL_2fb77ac98cad4c18b470e559162073f0",
              "IPY_MODEL_e014f9a4858846c59011cdb018b159b3"
            ],
            "layout": "IPY_MODEL_92a5b5585b6f4c7f951873a179783fc1"
          }
        },
        "580d0772992f4b23b60dc4c953f5689e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f13cabd1731a44479caa7d81c354a72f",
            "placeholder": "​",
            "style": "IPY_MODEL_6fdc7fb3edeb424cb804be209a696f6d",
            "value": "Downloading: 100%"
          }
        },
        "2fb77ac98cad4c18b470e559162073f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_84fc7f888db3419386f6a1cbae7b47b4",
            "max": 1197,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_74f28693b0104db795df1c3c77212648",
            "value": 1197
          }
        },
        "e014f9a4858846c59011cdb018b159b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_157a6c9865304d31825ef87d9d63f247",
            "placeholder": "​",
            "style": "IPY_MODEL_73b0c860a8f14be5a8e66b976f4733a8",
            "value": " 1.20k/1.20k [00:00&lt;00:00, 17.8kB/s]"
          }
        },
        "92a5b5585b6f4c7f951873a179783fc1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f13cabd1731a44479caa7d81c354a72f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6fdc7fb3edeb424cb804be209a696f6d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "84fc7f888db3419386f6a1cbae7b47b4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "74f28693b0104db795df1c3c77212648": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "157a6c9865304d31825ef87d9d63f247": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "73b0c860a8f14be5a8e66b976f4733a8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "089164aa48554aa7b3ff01d3a0fccfc8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8700cc3d1ae049f9ae8ae1e858ad0e9c",
              "IPY_MODEL_d7d5fefc19694eb5830cef884f7de329",
              "IPY_MODEL_19db6b5db0a2424db2c9ee4903f06b78"
            ],
            "layout": "IPY_MODEL_0ad91a0e58674d8abff81735fd9fe5d8"
          }
        },
        "8700cc3d1ae049f9ae8ae1e858ad0e9c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ca028bd98e694a548d4ba736ba04e1e5",
            "placeholder": "​",
            "style": "IPY_MODEL_dffc9269c2d14d8ca75ea0cb2e35c268",
            "value": "Downloading: 100%"
          }
        },
        "d7d5fefc19694eb5830cef884f7de329": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5687efe15e2e47f3a37e3fdfd0f33e77",
            "max": 242065649,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a2e43eeb736045fe9393d684286a57eb",
            "value": 242065649
          }
        },
        "19db6b5db0a2424db2c9ee4903f06b78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d6da3da348e7435b8fae411d83d4172a",
            "placeholder": "​",
            "style": "IPY_MODEL_e8ad092dc8764c4793f094372a02dfe1",
            "value": " 242M/242M [00:11&lt;00:00, 24.0MB/s]"
          }
        },
        "0ad91a0e58674d8abff81735fd9fe5d8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ca028bd98e694a548d4ba736ba04e1e5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dffc9269c2d14d8ca75ea0cb2e35c268": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5687efe15e2e47f3a37e3fdfd0f33e77": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a2e43eeb736045fe9393d684286a57eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d6da3da348e7435b8fae411d83d4172a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e8ad092dc8764c4793f094372a02dfe1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5da79695e4ea486bae4babfa85674f7c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5ab2d1fed49a42b98308af92cb2f38fd",
              "IPY_MODEL_08e2f646fce4452d913a2634290f7bc8",
              "IPY_MODEL_bcb6aa8fc34e498392b1129802203ae6"
            ],
            "layout": "IPY_MODEL_d6fc5c8257bb4408ba6bd3aced92fcef"
          }
        },
        "5ab2d1fed49a42b98308af92cb2f38fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b365cb0f0d024f90b84dbe7db9ab27df",
            "placeholder": "​",
            "style": "IPY_MODEL_60a806da87fb45daa41bffb0d0f8045b",
            "value": "Downloading: 100%"
          }
        },
        "08e2f646fce4452d913a2634290f7bc8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_29f8556d9c1041869ce722676346a54e",
            "max": 791656,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_fceb00b43c54427daefef09b78eba5ba",
            "value": 791656
          }
        },
        "bcb6aa8fc34e498392b1129802203ae6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_756d24acd1a74c0ba17b561e5a43711c",
            "placeholder": "​",
            "style": "IPY_MODEL_b5a5fa414ffb4b0b95c0b2d08245b8b0",
            "value": " 792k/792k [00:00&lt;00:00, 627kB/s]"
          }
        },
        "d6fc5c8257bb4408ba6bd3aced92fcef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b365cb0f0d024f90b84dbe7db9ab27df": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "60a806da87fb45daa41bffb0d0f8045b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "29f8556d9c1041869ce722676346a54e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fceb00b43c54427daefef09b78eba5ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "756d24acd1a74c0ba17b561e5a43711c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b5a5fa414ffb4b0b95c0b2d08245b8b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/nikitachaudharicodes/Research-Paper-Summarizer/blob/main/T5.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i9nzqpEJWBCM",
        "outputId": "39261f78-d1dc-466c-c5d3-e43143d5fa30"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (1.12.1+cu113)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch) (4.1.1)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.24.0-py3-none-any.whl (5.5 MB)\n",
            "\u001b[K     |████████████████████████████████| 5.5 MB 4.2 MB/s \n",
            "\u001b[?25hCollecting huggingface-hub<1.0,>=0.10.0\n",
            "  Downloading huggingface_hub-0.10.1-py3-none-any.whl (163 kB)\n",
            "\u001b[K     |████████████████████████████████| 163 kB 44.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 7.6 MB 43.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.8.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.13.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.1)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.10.0->transformers) (4.1.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.10.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.9.24)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Installing collected packages: tokenizers, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.10.1 tokenizers-0.13.2 transformers-4.24.0\n"
          ]
        }
      ],
      "source": [
        "!pip install torch\n",
        "!pip install transformers"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "T5\n",
        "\n"
      ],
      "metadata": {
        "id": "4-L0j6Yy89SA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch"
      ],
      "metadata": {
        "id": "RsgKX2Fb8Lyo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n"
      ],
      "metadata": {
        "id": "YcP9O9QA8P0q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install Sentencepiece"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ehz1oU3j89iy",
        "outputId": "584b7c8d-84c1-4c45-ce60-7772c282db48"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting Sentencepiece\n",
            "  Downloading sentencepiece-0.1.97-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.3 MB 4.2 MB/s \n",
            "\u001b[?25hInstalling collected packages: Sentencepiece\n",
            "Successfully installed Sentencepiece-0.1.97\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install sentencepiece"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eCbp69AV_u8A",
        "outputId": "cf6187c4-ed0d-482b-aff2-ceb4b1dd66c0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.7/dist-packages (0.1.97)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import T5Tokenizer, T5ForConditionalGeneration, T5Config\n"
      ],
      "metadata": {
        "id": "SoiVCKtv8TvR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = T5ForConditionalGeneration.from_pretrained('t5-small')\n"
      ],
      "metadata": {
        "id": "y3P2YmUF8XZA",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81,
          "referenced_widgets": [
            "27fb5c67b0fa4b0dac53106f4ec76c45",
            "580d0772992f4b23b60dc4c953f5689e",
            "2fb77ac98cad4c18b470e559162073f0",
            "e014f9a4858846c59011cdb018b159b3",
            "92a5b5585b6f4c7f951873a179783fc1",
            "f13cabd1731a44479caa7d81c354a72f",
            "6fdc7fb3edeb424cb804be209a696f6d",
            "84fc7f888db3419386f6a1cbae7b47b4",
            "74f28693b0104db795df1c3c77212648",
            "157a6c9865304d31825ef87d9d63f247",
            "73b0c860a8f14be5a8e66b976f4733a8",
            "089164aa48554aa7b3ff01d3a0fccfc8",
            "8700cc3d1ae049f9ae8ae1e858ad0e9c",
            "d7d5fefc19694eb5830cef884f7de329",
            "19db6b5db0a2424db2c9ee4903f06b78",
            "0ad91a0e58674d8abff81735fd9fe5d8",
            "ca028bd98e694a548d4ba736ba04e1e5",
            "dffc9269c2d14d8ca75ea0cb2e35c268",
            "5687efe15e2e47f3a37e3fdfd0f33e77",
            "a2e43eeb736045fe9393d684286a57eb",
            "d6da3da348e7435b8fae411d83d4172a",
            "e8ad092dc8764c4793f094372a02dfe1"
          ]
        },
        "outputId": "c469b63a-7d22-4b3e-af1d-e76f167bf9f9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.20k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "27fb5c67b0fa4b0dac53106f4ec76c45"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/242M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "089164aa48554aa7b3ff01d3a0fccfc8"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer= T5Tokenizer.from_pretrained('t5-small')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173,
          "referenced_widgets": [
            "5da79695e4ea486bae4babfa85674f7c",
            "5ab2d1fed49a42b98308af92cb2f38fd",
            "08e2f646fce4452d913a2634290f7bc8",
            "bcb6aa8fc34e498392b1129802203ae6",
            "d6fc5c8257bb4408ba6bd3aced92fcef",
            "b365cb0f0d024f90b84dbe7db9ab27df",
            "60a806da87fb45daa41bffb0d0f8045b",
            "29f8556d9c1041869ce722676346a54e",
            "fceb00b43c54427daefef09b78eba5ba",
            "756d24acd1a74c0ba17b561e5a43711c",
            "b5a5fa414ffb4b0b95c0b2d08245b8b0"
          ]
        },
        "id": "v5tiJjXa8Zw1",
        "outputId": "6618f0cc-851b-42d7-ad11-cac9704e451e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/792k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5da79695e4ea486bae4babfa85674f7c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/models/t5/tokenization_t5.py:174: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5.\n",
            "For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`.\n",
            "- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding.\n",
            "- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding.\n",
            "- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value.\n",
            "  FutureWarning,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd"
      ],
      "metadata": {
        "id": "q8558Q-MMmXG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df=pd.read_csv(\"/content/papers1 - Sheet1.csv\")"
      ],
      "metadata": {
        "id": "PCQsqGOYMmhc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 622
        },
        "id": "3_ZlmZleMmmK",
        "outputId": "a574072e-e289-49f2-8cc9-c94834868d94"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                               TITLE  \\\n",
              "0  Stock Prediction using Machine Learning a Revi...   \n",
              "1  Sentiment Analysis for Product Recommendation ...   \n",
              "2  EDM – survey of performance factors and algori...   \n",
              "3                            Fake News Detector: FND   \n",
              "4  Effect of image binarization thresholds on bre...   \n",
              "5  A review of microscopic analysis of blood cell...   \n",
              "6  On Context Awareness for Multisensor Data Fusi...   \n",
              "7  Iris Liveness Detection for Biometric Authenti...   \n",
              "\n",
              "                                        INTRODUCTION  \\\n",
              "0  Machine learning can be defined as the data wh...   \n",
              "1  Sentimental Analysis is nothing but the task o...   \n",
              "2  Data mining is a process to extract informatio...   \n",
              "3  Fake news can be defined as a pseudo-news or j...   \n",
              "4  Early detection of breast cancer plays a very ...   \n",
              "5  Blood, the most integral part of the body, is ...   \n",
              "6  The concept of the internet of things (IoT) or...   \n",
              "7  During the primeval eras, there were restricte...   \n",
              "\n",
              "                                               Ref 1  \\\n",
              "0  In machine learning, data is obtained by knowl...   \n",
              "1  Sentiment analysis is computational methodolog...   \n",
              "2  Educational Data Mining refers to techniques, ...   \n",
              "3  Parameters including websites data, website va...   \n",
              "4                                                NaN   \n",
              "5                                                NaN   \n",
              "6                                                NaN   \n",
              "7                                                NaN   \n",
              "\n",
              "                                               Ref 2  \\\n",
              "0  Machine learning is used to describe the proce...   \n",
              "1  This paper presents a computational approach f...   \n",
              "2  Data Mining is the extraction of information f...   \n",
              "3  Rapid spread of fake news is a serious problem...   \n",
              "4                                                NaN   \n",
              "5                                                NaN   \n",
              "6                                                NaN   \n",
              "7                                                NaN   \n",
              "\n",
              "                                               Ref 3  \n",
              "0  Machine learning can be defined as the data wh...  \n",
              "1  Sentimental Analysis is a task of Natural Lang...  \n",
              "2  Data mining is a process to extracting informa...  \n",
              "3  Fake news, defined as a pseudo-news or junk ne...  \n",
              "4                                                NaN  \n",
              "5                                                NaN  \n",
              "6                                                NaN  \n",
              "7                                                NaN  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-79b5a6b4-1fb3-4eab-aa22-310c38024975\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>TITLE</th>\n",
              "      <th>INTRODUCTION</th>\n",
              "      <th>Ref 1</th>\n",
              "      <th>Ref 2</th>\n",
              "      <th>Ref 3</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Stock Prediction using Machine Learning a Revi...</td>\n",
              "      <td>Machine learning can be defined as the data wh...</td>\n",
              "      <td>In machine learning, data is obtained by knowl...</td>\n",
              "      <td>Machine learning is used to describe the proce...</td>\n",
              "      <td>Machine learning can be defined as the data wh...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Sentiment Analysis for Product Recommendation ...</td>\n",
              "      <td>Sentimental Analysis is nothing but the task o...</td>\n",
              "      <td>Sentiment analysis is computational methodolog...</td>\n",
              "      <td>This paper presents a computational approach f...</td>\n",
              "      <td>Sentimental Analysis is a task of Natural Lang...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>EDM – survey of performance factors and algori...</td>\n",
              "      <td>Data mining is a process to extract informatio...</td>\n",
              "      <td>Educational Data Mining refers to techniques, ...</td>\n",
              "      <td>Data Mining is the extraction of information f...</td>\n",
              "      <td>Data mining is a process to extracting informa...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Fake News Detector: FND</td>\n",
              "      <td>Fake news can be defined as a pseudo-news or j...</td>\n",
              "      <td>Parameters including websites data, website va...</td>\n",
              "      <td>Rapid spread of fake news is a serious problem...</td>\n",
              "      <td>Fake news, defined as a pseudo-news or junk ne...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Effect of image binarization thresholds on bre...</td>\n",
              "      <td>Early detection of breast cancer plays a very ...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>A review of microscopic analysis of blood cell...</td>\n",
              "      <td>Blood, the most integral part of the body, is ...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>On Context Awareness for Multisensor Data Fusi...</td>\n",
              "      <td>The concept of the internet of things (IoT) or...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>Iris Liveness Detection for Biometric Authenti...</td>\n",
              "      <td>During the primeval eras, there were restricte...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-79b5a6b4-1fb3-4eab-aa22-310c38024975')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-79b5a6b4-1fb3-4eab-aa22-310c38024975 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-79b5a6b4-1fb3-4eab-aa22-310c38024975');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from nltk.translate.bleu_score import corpus_bleu, sentence_bleu\n"
      ],
      "metadata": {
        "id": "EUjWHKbbMmwj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iLIPACEa-jYP"
      },
      "outputs": [],
      "source": [
        "reference11=df['Ref 1'][0]\n",
        "reference21=df['Ref 1'][1]\n",
        "reference31=df['Ref 1'][2]\n",
        "reference41=df['Ref 1'][3]\n",
        "#print(reference11, reference21, reference31, reference41)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3B-byrbQ-jaw"
      },
      "outputs": [],
      "source": [
        "text1=df['INTRODUCTION'][0]\n",
        "text2=df['INTRODUCTION'][1]\n",
        "text3=df['INTRODUCTION'][2]\n",
        "text4=df['INTRODUCTION'][3]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lGXXpJnS-jd0"
      },
      "outputs": [],
      "source": [
        "reference12=df['Ref 2'][0]\n",
        "reference22=df['Ref 2'][1]\n",
        "reference32=df['Ref 2'][2]\n",
        "reference42=df['Ref 2'][3]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9Nc384u5_io4"
      },
      "outputs": [],
      "source": [
        "reference13=df['Ref 3'][0]\n",
        "reference23=df['Ref 3'][1]\n",
        "reference33=df['Ref 3'][2]\n",
        "reference43=df['Ref 3'][3]"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Sample1"
      ],
      "metadata": {
        "id": "7qIhbJT5eAxG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "text1 "
      ],
      "metadata": {
        "id": "IWxBlzzN8b2b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 174
        },
        "outputId": "1e81f888-5fe1-467d-8944-83adcba91f35"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Machine learning can be defined as the data which is obtained by knowledge extraction. Machines don‟t have to be programmed explicitly instead they are trained to make decisions that are driven by data. Instead of writing a code for every specific problem, data is provided to the generic algorithms and logic is developed on the basis of that data. When a machine improves its performance based on its past experiences it can be said that machine has truly learnt. The technique for most accurate prediction is by learning from past instances, and to make a program to do this is best possible with machine learning techniques. Any machine learning technique (supervised or unsupervised) is efficient enough to generate rules for programs, in consideration with present ones to take a better decision. In this scenario, the decision is whether the stock will increase or decrease (Stock analysis).'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "preprocess_text1 = text1.strip().replace(\"\\n\",\"\")"
      ],
      "metadata": {
        "id": "wZAWdCoL_7qj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "t5_prepared_Text1 = \"summarize: \"+preprocess_text1\n"
      ],
      "metadata": {
        "id": "OQJhnhyR_-QH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print (\"original text preprocessed: \\n\", preprocess_text1)\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GbWiz2POABu0",
        "outputId": "975ad245-4a48-4dd4-9645-2d3809260083"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "original text preprocessed: \n",
            " Machine learning can be defined as the data which is obtained by knowledge extraction. Machines don‟t have to be programmed explicitly instead they are trained to make decisions that are driven by data. Instead of writing a code for every specific problem, data is provided to the generic algorithms and logic is developed on the basis of that data. When a machine improves its performance based on its past experiences it can be said that machine has truly learnt. The technique for most accurate prediction is by learning from past instances, and to make a program to do this is best possible with machine learning techniques. Any machine learning technique (supervised or unsupervised) is efficient enough to generate rules for programs, in consideration with present ones to take a better decision. In this scenario, the decision is whether the stock will increase or decrease (Stock analysis).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_text1 = tokenizer.encode(t5_prepared_Text1, return_tensors=\"pt\")\n"
      ],
      "metadata": {
        "id": "EsuuqnwCAEO6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "summary1 = model.generate(tokenized_text1,\n",
        "                                    num_beams=4,\n",
        "                                    no_repeat_ngram_size=2,\n",
        "                                    min_length=30,\n",
        "                                    max_length=100,\n",
        "                                    early_stopping=True)"
      ],
      "metadata": {
        "id": "1Kd9f4FRAIcW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "output1 = tokenizer.decode(summary1[0], skip_special_tokens=True)\n"
      ],
      "metadata": {
        "id": "qmoGD8mpANYo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print (\"\\n\\nSummarized text: \\n\",output1)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JKFYp3iJAP29",
        "outputId": "b7859725-f164-42c9-9eb0-64a46c57cdb2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "Summarized text: \n",
            " machine learning can be defined as the data which is obtained by knowledge extraction. the technique for most accurate prediction is by learning from past instances, and to make a program to do this is best possible with machines learning techniques (supervised or unsupervised) the decision is whether the stock will increase or decrease (Stock analysis).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install rouge"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XV5Mu2huAly2",
        "outputId": "b5419f1c-0ede-4add-dedb-5f701ce0f154"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting rouge\n",
            "  Downloading rouge-1.0.1-py3-none-any.whl (13 kB)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from rouge) (1.15.0)\n",
            "Installing collected packages: rouge\n",
            "Successfully installed rouge-1.0.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from rouge import Rouge"
      ],
      "metadata": {
        "id": "lCL_OxUqAnnU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "reference1 =\"In machine learning, data is obtained by knowledge extraction. Machine learning techniques learn from past predictions and give an outcome. \"\n",
        "rouge = Rouge()\n",
        "rouge.get_scores(output1, reference11)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yBoIb_sXASPN",
        "outputId": "1987e73b-1f22-4132-fce5-8e0caece2ba9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'rouge-1': {'r': 0.6, 'p': 0.26666666666666666, 'f': 0.36923076497041424},\n",
              "  'rouge-2': {'r': 0.3157894736842105,\n",
              "   'p': 0.10909090909090909,\n",
              "   'f': 0.16216215834550776},\n",
              "  'rouge-l': {'r': 0.55, 'p': 0.24444444444444444, 'f': 0.33846153420118347}}]"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "SqgCqKONK2_G"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I1YfAnQTeq4D",
        "outputId": "6b305450-3ff2-424a-e0c7-6e54fd16050b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "BLEU SCORE: 0.24225366455985492\n"
          ]
        }
      ],
      "source": [
        "reference_1 = [\n",
        "    reference12.split(),\n",
        "    reference13.split()\n",
        "]\n",
        "from nltk.translate.bleu_score import corpus_bleu\n",
        "summary_1=output1.split()\n",
        "score = sentence_bleu(reference_1, summary_1)\n",
        "print(\"BLEU SCORE:\",score)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer.model_max_length"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gLFgSQLtAf9p",
        "outputId": "eb4b627e-1ecc-47b7-9b4a-e6558c391d25"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "512"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(tokenizer(text1)['input_ids'])\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Iwb9kmwYBmX3",
        "outputId": "b1917836-5df1-471d-ec57-84f6bdae5e9b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "175"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Sample 2"
      ],
      "metadata": {
        "id": "sO850mHEMH-O"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#sample 2: Sentiment\n",
        "text2 "
      ],
      "metadata": {
        "id": "ktNW_bbvBtTV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6a181d95-fbd4-44ae-95b8-10ca88889458"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Sentimental Analysis is nothing but the task of Natural Language Processing. It observes the attitude of customer behind the comments. Sentiment analysis is a method of identifying sentiments in text.  Sentiment analysis is computational methodology of extracting sentiments from text, speech or dataset. It can classify emotions, attitude, opinion and subjective impression into polarity. Researchers and decision makers understand the approach of the people using consumer sentiment analysis and can make decisions accordingly. Business analysis application can be developed by using this technique. Social network development and popularity is increasing day by day. A growing number of users prefer to order online products and prefer to share their experiences on social networks. Searching for appropriate product online is a difficult task. Recommender system can help users by providing suggestions. Recommender system creates recommendation list. There are three Recommendation system approaches, content based, collaborative and hybrid approach. The content-based approach considers the information of an item and the user’s profile and the recommendation of items is based on the user’s preferences. The collaborative-based approach analyses the user behaviour and preferences and find the same preferences among people. It is well known that collaborative-based techniques are normally more accurate than content-based techniques. The hybrid approach combines both methods. User’s feedback is important tool in recommendation system. The recommendation systems gives suggestion based on the user history and on the user’s profile, but nowadays, the recommendations are starting to explore making more suggestions based on sentiment analysis.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "preprocess_text2 = text2.strip().replace(\"\\n\",\"\")\n"
      ],
      "metadata": {
        "id": "80Rwh-fP7TzR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "t5_prepared_Text2 = \"summarize: \"+preprocess_text2\n"
      ],
      "metadata": {
        "id": "Mrw77qg27dhe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_text2 = tokenizer.encode(t5_prepared_Text2, return_tensors=\"pt\")\n"
      ],
      "metadata": {
        "id": "V9txPR3v7gZy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "summary2 = model.generate(tokenized_text2,\n",
        "                                    num_beams=4,\n",
        "                                    no_repeat_ngram_size=2,\n",
        "                                    min_length=30,\n",
        "                                    max_length=150,\n",
        "                                    early_stopping=True)"
      ],
      "metadata": {
        "id": "R-RfqvTw7YET"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "output2 = tokenizer.decode(summary2[0], skip_special_tokens=True)\n"
      ],
      "metadata": {
        "id": "3W9l1eEM7jnL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print (\"\\n\\nSummarized text: \\n\",output2)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oGGxZQ9j7nCz",
        "outputId": "ef9988d2-7760-4b6e-8a15-7b032deb2ce9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "Summarized text: \n",
            " Sentiment analysis is a computational method of extracting sentiments from text, speech or dataset. it can classify emotions, attitude, opinion and subjective impression into polarity. researchers and decision makers understand the approach of people using consumer sentiment analysis and can make decisions accordingly.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "rouge = Rouge()\n",
        "rouge.get_scores(output2, reference21)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U1pSKjda8DaW",
        "outputId": "2db73ea3-23d9-4ce5-ca34-c7d6e6672861"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'rouge-1': {'r': 0.8695652173913043,\n",
              "   'p': 0.5263157894736842,\n",
              "   'f': 0.6557377002203709},\n",
              "  'rouge-2': {'r': 0.6818181818181818,\n",
              "   'p': 0.35714285714285715,\n",
              "   'f': 0.46874999548828133},\n",
              "  'rouge-l': {'r': 0.8695652173913043,\n",
              "   'p': 0.5263157894736842,\n",
              "   'f': 0.6557377002203709}}]"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d-IzmRG8DCZv",
        "outputId": "4a05d607-0123-4d84-f797-69aaf598b2ea"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "BLEU SCORE: 0.22857546269013312\n"
          ]
        }
      ],
      "source": [
        "reference_2 = [\n",
        "    reference22.split(),\n",
        "    reference23.split()\n",
        "]\n",
        "from nltk.translate.bleu_score import corpus_bleu\n",
        "summary_2=output2.split()\n",
        "score = sentence_bleu(reference_2, summary_2)\n",
        "print(\"BLEU SCORE:\",score)"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "vUEXHSR47ZQE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Sample 3"
      ],
      "metadata": {
        "id": "YANEbB3DMOH2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#sample 3: EDM\n",
        "text3 "
      ],
      "metadata": {
        "id": "YcGcClhMMPUH",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "30154895-592d-4a13-f681-e046156480a1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Data mining is a process to extract information from a data set and transform it into an understandable structure for further use. Educational Data Mining (EDM) relates to the inter-disciplinary research that deals with the development of various methods and techniques to explore the data generated from different educational sources. Analysing educational data could provide information of student’s behaviours, based on which education policies would be made properly.[1] Educational Data Mining refers to techniques, tools, and research designed for automatically extracting meaning from large repositories of data generated by or related to people’s learning activities in educational settings. Recent advances in educational technology, including the increase in computing power and the ability to log fine-grained data about student’s use of a computer-based learning environment, have led to an increased interest in developing techniques for analysing the large amounts of data generated in educational settings. This paper presents a study on current state of EDM and identifies the algorithms applied. Section 2 talks about the Goals and Methods in EDM, section 3 proposes purpose of study and presents the study in summarized fashion. Section 4 talks about role and study about performance factors. Section 5 presents the findings from the survey followed by conclusion. '"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "preprocess_text3 = text3.strip().replace(\"\\n\",\"\")\n"
      ],
      "metadata": {
        "id": "_W54mkNSMhAs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "t5_prepared_Text3 = \"summarize: \"+preprocess_text3\n"
      ],
      "metadata": {
        "id": "tR5qdU8ZMhEM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_text3 = tokenizer.encode(t5_prepared_Text3, return_tensors=\"pt\")\n"
      ],
      "metadata": {
        "id": "S8cj1hgbfLtH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "summary3 = model.generate(tokenized_text3,\n",
        "                                    num_beams=4,\n",
        "                                    no_repeat_ngram_size=2,\n",
        "                                    min_length=30,\n",
        "                                    max_length=150,\n",
        "                                    early_stopping=True)"
      ],
      "metadata": {
        "id": "zgmDvMgWdBle"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "output3 = tokenizer.decode(summary3[0], skip_special_tokens=True)\n"
      ],
      "metadata": {
        "id": "LaBCJHCVdBod"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print (\"\\n\\nSummarized text: \\n\",output3)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PyayKTDmdBsr",
        "outputId": "280e21f3-c68b-4a8b-bcd9-70bad7e6ba0d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "Summarized text: \n",
            " educational data mining refers to techniques, tools, and research designed for automatically extracting meaning from large repositories of data generated by or related to people’s learning activities.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "7srnBqVOdVsB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "7RUcySctdVvO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rouge = Rouge()\n",
        "rouge.get_scores(output3, reference31)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LLSdz_pZMhGv",
        "outputId": "b08f0ea0-72c1-4982-b0f5-157c7f0f835f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'rouge-1': {'r': 0.6, 'p': 0.72, 'f': 0.654545449586777},\n",
              "  'rouge-2': {'r': 0.5333333333333333,\n",
              "   'p': 0.6153846153846154,\n",
              "   'f': 0.5714285664540818},\n",
              "  'rouge-l': {'r': 0.5666666666666667, 'p': 0.68, 'f': 0.6181818132231407}}]"
            ]
          },
          "metadata": {},
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TCQjRCw9BBI4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d159377f-8b0b-4900-9a2f-06d2740d31e1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "BLEU SCORE: 3.6613309501685013e-155\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/nltk/translate/bleu_score.py:552: UserWarning: \n",
            "The hypothesis contains 0 counts of 3-gram overlaps.\n",
            "Therefore the BLEU score evaluates to 0, independently of\n",
            "how many N-gram overlaps of lower order it contains.\n",
            "Consider using lower n-gram order or use SmoothingFunction()\n",
            "  warnings.warn(_msg)\n",
            "/usr/local/lib/python3.7/dist-packages/nltk/translate/bleu_score.py:552: UserWarning: \n",
            "The hypothesis contains 0 counts of 4-gram overlaps.\n",
            "Therefore the BLEU score evaluates to 0, independently of\n",
            "how many N-gram overlaps of lower order it contains.\n",
            "Consider using lower n-gram order or use SmoothingFunction()\n",
            "  warnings.warn(_msg)\n"
          ]
        }
      ],
      "source": [
        "reference_3 = [\n",
        "    reference32.split(),\n",
        "    reference33.split()\n",
        "]\n",
        "from nltk.translate.bleu_score import corpus_bleu\n",
        "summary_3=output3.split()\n",
        "score = sentence_bleu(reference_3, summary_3)\n",
        "print(\"BLEU SCORE:\",score)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Sample 4"
      ],
      "metadata": {
        "id": "hZnqwmw4MPpm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#sample 4 : Fake News\n",
        "text4 "
      ],
      "metadata": {
        "id": "W9xufZu_MQg1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6617d1f3-c7ad-43e5-b43f-bab2d79a87b6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Fake news can be defined as a pseudo-news or junk news which is false information or content spread via different broadcasting media. This false information is mainly created to mislead people or to cause ravage to a person, organization, etc., or to gain financial benefits. Easier ways to get news nowadays are from sources like media outlets, newspapers, journalists where they follow defined strict codes of practice; this is the traditional method. But the increase in internet network has changed the certain aspect of publishing and sharing the news and information with less editorial standards and regulations. On the other hand, the news is being manipulated by various networking sites based on personal opinions or interests.[1] Fake news causes immense damage in different ways such as misleading people with sharing false information, this can create racists ideas or can damage people’s sentiments, which can give rise to violence among people i.e. causing real-life impacts, etc. This rapid spread of fake news is a serious problem calling for AI solutions.[3] To overcome this, the system is developed which identifies the news authenticity and identifies it as Fake News or Real News. This is done by analyzing the news content where analysis is done on the basis of defined parameters. In the proposed system the user directly enters the data or the news content (news headlines) and enters into the systems search bar and gets the result. The proposed system works upon the two main algorithms i.e. Naive Bayes algorithm and TFIDF algorithm. The defined parameters help to identify the news content and increases the accuracy of the system. Parameters considered are website data, website validates, the similarity of content, timestamps, reviews, and grammatical analysis. These parameters calculate the news authenticity and generate the final result as fake or real news.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "preprocess_text4 = text4.strip().replace(\"\\n\",\"\")\n"
      ],
      "metadata": {
        "id": "n53oM3pPMmQ7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "t5_prepared_Text4 = \"summarize: \"+preprocess_text4\n"
      ],
      "metadata": {
        "id": "dKYiduO1MmT6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_text4 = tokenizer.encode(t5_prepared_Text4, return_tensors=\"pt\")\n"
      ],
      "metadata": {
        "id": "9vZi3ToEfNtU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "summary4 = model.generate(tokenized_text4,\n",
        "                                    num_beams=4,\n",
        "                                    no_repeat_ngram_size=2,\n",
        "                                    min_length=30,\n",
        "                                    max_length=100,\n",
        "                                    early_stopping=True)"
      ],
      "metadata": {
        "id": "dLKXuB_5MmWG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "output4 = tokenizer.decode(summary4[0], skip_special_tokens=True)\n"
      ],
      "metadata": {
        "id": "p8H2EtM3MmaP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print (\"\\n\\nSummarized text: \\n\",output4)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2ZtOJl-PdO7n",
        "outputId": "0ede9750-af1b-4902-d4da-4d95619812fc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "Summarized text: \n",
            " fake news is mainly created to mislead people or to cause ravage to a person, organization, etc., or gain financial benefits. the rise in internet network has changed the certain aspect of publishing and sharing the news with less editorial standards and regulations based on personal opinions or interests. this can create racists ideas or can damage people’s sentiments, which can give rise to violence among people i.e. causing real-life impacts,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "tv_JrG0ddO-g"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rouge = Rouge()\n",
        "rouge.get_scores(output3, reference41)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sr3zfKIXMmdB",
        "outputId": "cc30ac2c-3c14-4858-b572-5b2052af36e3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'rouge-1': {'r': 0.07407407407407407, 'p': 0.08, 'f': 0.0769230719304737},\n",
              "  'rouge-2': {'r': 0.0, 'p': 0.0, 'f': 0.0},\n",
              "  'rouge-l': {'r': 0.07407407407407407, 'p': 0.08, 'f': 0.0769230719304737}}]"
            ]
          },
          "metadata": {},
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R1xCfmI9BQNf",
        "outputId": "5f187c51-e552-42d4-8195-6a5d49dbbda6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "BLEU SCORE: 0.09943862575374686\n"
          ]
        }
      ],
      "source": [
        "reference_4 = [\n",
        "    reference42.split(),\n",
        "    reference43.split()\n",
        "]\n",
        "from nltk.translate.bleu_score import corpus_bleu\n",
        "summary_4=output4.split()\n",
        "score = sentence_bleu(reference_4, summary_4)\n",
        "print(\"BLEU SCORE:\",score)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Sample 5"
      ],
      "metadata": {
        "id": "7qRRBPnNMQ3D"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#Sample 5: A review of microscopic analysis of blood cells for disease detection with AI perspective\n",
        "text5 = \"\"\"\n",
        "Blood, the most integral part of the body, is constituted of white blood cells (WBC), red blood cells (RBC), platelets, and plasma. \n",
        "This can be further categorized as; cells and platelets are about 45% of human blood, whereas the remaining 55% is filled by plasma (the yellow fluid in the blood). \n",
        "These components and their physical properties like size, shape, color, and count in the whole blood change due to ingress of any foreign object or micro-organism can lead to any sort of infections.\n",
        "\n",
        "There are different pathological procedures for the detection of diseases. In most cases, microscopic imaging plays a vital role in predicting and detecting abnormalities and occurrences of diseases within the body. \n",
        "Typically, the health of any person is judged by analyzing different features of blood cells and their counts.\n",
        "\n",
        "\"\"\""
      ],
      "metadata": {
        "id": "TLaq9cIBMRp3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "preprocess_text5 = text5.strip().replace(\"\\n\",\"\")\n"
      ],
      "metadata": {
        "id": "Hpglly1wMqe3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "t5_prepared_Text5 = \"summarize: \"+preprocess_text5\n"
      ],
      "metadata": {
        "id": "Q8Lb7kVpMqh_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_text5 = tokenizer.encode(t5_prepared_Text5, return_tensors=\"pt\")\n"
      ],
      "metadata": {
        "id": "8v72WmQzfSJd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "summary5 = model.generate(tokenized_text5,\n",
        "                                    num_beams=4,\n",
        "                                    no_repeat_ngram_size=2,\n",
        "                                    min_length=30,\n",
        "                                    max_length=100,\n",
        "                                    early_stopping=True)"
      ],
      "metadata": {
        "id": "Hj9nq3jeMqk9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "output5 = tokenizer.decode(summary5[0], skip_special_tokens=True)\n"
      ],
      "metadata": {
        "id": "0nOh-BXgMqoH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print (\"\\n\\nSummarized text: \\n\",output5)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lAiz7-4udPps",
        "outputId": "56dd865a-6b82-4945-820d-d31f994a8318"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "Summarized text: \n",
            " cells and platelets are about 45% of human blood, whereas the remaining 55% is filled by plasma (the yellow fluid in the blood) there are different pathological procedures for the detection of diseases.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "JNe1D7efdPti"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "reference5 =\"There are different pathological procedures for the detection of diseases. Typically, the health of any person is judged by analyzing different features of blood cells and their counts. In most cases, microscopic imaging plays a vital role in predicting and detecting abnormalities and occurrences of diseases within the body. \"\n",
        "rouge = Rouge()\n",
        "rouge.get_scores(output3, reference5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IJaouZLVMqq6",
        "outputId": "0601c28e-a41e-4aa6-b82f-5c506e7c6b03"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'rouge-1': {'r': 0.0975609756097561, 'p': 0.16, 'f': 0.12121211650596897},\n",
              "  'rouge-2': {'r': 0.0, 'p': 0.0, 'f': 0.0},\n",
              "  'rouge-l': {'r': 0.0975609756097561, 'p': 0.16, 'f': 0.12121211650596897}}]"
            ]
          },
          "metadata": {},
          "execution_count": 65
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Sample 6"
      ],
      "metadata": {
        "id": "j84P9cgNMSMU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#Sample 6: On Context Awareness for Multisensor Data Fusion in IoT\n",
        "text6 = \"\"\"\n",
        "The concept of the internet of things (IoT) originated in the Auto-ID Center at the Massachusetts Institute of Technology in 1999.\n",
        "Kevin Ashton had imagined a world in which all electronic devices are networked and every object, whether physical or electronic, is electronically tagged with information applicable to that object.\n",
        "The underlying aim of this concept is the achievement of pervasive connections between the internet and objects around us. \n",
        "It is perfect assimilation of real-world objects with logical things.\n",
        "Multi-sensor data fusion system is analogous to human who can sense the environment with the help of their sensory organs like nose, ears, skin, etc. and make correct inferences about their surroundings.\n",
        "Multisensor data fusion refers to the comprehensive fusing of sensory data from multiple sensors and related information in order to provide more reliable and accurate information that could be achieved using a single, independent sensor.\n",
        "Sensor fusion technology was primarily developed for Military surveillance research and robotics by US DoD. \n",
        "Later, it has got commercially wider acceptance in the areas, such as intelligent transport system, geographic information, land and ocean surveillance, robotics, data and information security, medical surveillance, diagnosis, etc.\n",
        " The only way to gain the required amount of information with the expected intelligence is viable with the help of multisensors data fusion approach.\n",
        "\n",
        "\"\"\""
      ],
      "metadata": {
        "id": "y1kPavo0MS_H"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "preprocess_text6 = text6.strip().replace(\"\\n\",\"\")\n"
      ],
      "metadata": {
        "id": "SWgtWFBIMxK3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "t5_prepared_Text6 = \"summarize: \"+preprocess_text6\n"
      ],
      "metadata": {
        "id": "hf6o3mIAMxOB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_text6 = tokenizer.encode(t5_prepared_Text6, return_tensors=\"pt\")\n"
      ],
      "metadata": {
        "id": "k73j3rHxfUcK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "summary6 = model.generate(tokenized_text6,\n",
        "                                    num_beams=4,\n",
        "                                    no_repeat_ngram_size=2,\n",
        "                                    min_length=30,\n",
        "                                    max_length=100,\n",
        "                                    early_stopping=True)"
      ],
      "metadata": {
        "id": "auhDVlEaMxQJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "output6 = tokenizer.decode(summary6[0], skip_special_tokens=True)\n"
      ],
      "metadata": {
        "id": "hlk4IDyodKPu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print (\"\\n\\nSummarized text: \\n\",output6)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ohZvCxRQdQhc",
        "outputId": "b61177f0-476f-41c3-fad8-f94e860d568b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "Summarized text: \n",
            " the concept of the internet of things originated in the auto-ID center at the Massachusetts Institute of Technology in 1999. it is perfect assimilation of real-world objects with logical things.multisensor data fusion system is analogous to human who can sense the environment with the help of their sensory organs like nose, ears, skin, etc.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "w6PodeDRdQkp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "B7nUKLl7dKU7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "reference6 =\"The aim of the internet of things is to achieve pervasive connections between the internet and objects around us. Multisensor data fusion refers to the comprehensive fusing of sensory data from multiple sensors. Sensor fusion technology was primarily developed for Military surveillance research and robotics by US DoD. \"\n",
        "rouge = Rouge()\n",
        "rouge.get_scores(output3, reference6)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q_HJtTcVMxUP",
        "outputId": "ccdc2d3f-061c-43be-bf39-cff729a97942"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'rouge-1': {'r': 0.225, 'p': 0.36, 'f': 0.2769230721893492},\n",
              "  'rouge-2': {'r': 0.02127659574468085,\n",
              "   'p': 0.038461538461538464,\n",
              "   'f': 0.02739725568774706},\n",
              "  'rouge-l': {'r': 0.15, 'p': 0.24, 'f': 0.1846153798816569}}]"
            ]
          },
          "metadata": {},
          "execution_count": 73
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Sample 7"
      ],
      "metadata": {
        "id": "XG1L1g2PMUdY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#sample 7: Fake News Detector: FND\n",
        "\n",
        "text7='''\n",
        "Fake news can be defined as a pseudo-news or junk news which is false information or content spread via different broadcasting media.\n",
        "This false information is mainly created to mislead people or to cause ravage to a person, organization, etc., or to gain financial benefits. \n",
        "Easier ways to get news nowadays are from sources like media outlets, newspapers, journalists where they follow defined strict codes of practice; this is the traditional method. \n",
        "But the increase in internet network has changed the certain aspect of publishing and sharing the news and information with less editorial standards and regulations.\n",
        "On the other hand, the news is being manipulated by various networking sites based on personal opinions or interests.\n",
        "Fake news causes immense damage in different ways such as misleading people with sharing false information, this can create racists ideas or can damage people’s sentiments, which can give rise to violence among people i.e. causing real-life impacts, etc. \n",
        "This rapid spread of fake news is a serious problem calling for AI solutions. To overcome this, the system is developed which identifies the news authenticity and identifies it as Fake News or Real News. \n",
        "This is done by analyzing the news content where analysis is done on the basis of defined parameters. \n",
        "In the proposed system the user directly enters the data or the news content (news headlines) and enters into the systems search bar and gets the result. \n",
        "The proposed system works upon the two main algorithms i.e. Naive Bayes algorithm and TFIDF algorithm. The defined parameters help to identify the news content and increases the accuracy of the system. \n",
        "Parameters considered are website data, website validates, the similarity of content, timestamps, reviews, and grammatical analysis. \n",
        "These parameters calculate the news authenticity and generate the final result as fake or real news.\n",
        "'''"
      ],
      "metadata": {
        "id": "QXhFHzwHMVVD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "preprocess_text7 = text7.strip().replace(\"\\n\",\"\")\n"
      ],
      "metadata": {
        "id": "g_0_An5nM2Mo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "t5_prepared_Text7 = \"summarize: \"+preprocess_text7\n"
      ],
      "metadata": {
        "id": "7pF0H3aNM2RN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_text7 = tokenizer.encode(t5_prepared_Text7, return_tensors=\"pt\")\n"
      ],
      "metadata": {
        "id": "1sjGiksdfXX8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "summary7 = model.generate(tokenized_text7,\n",
        "                                    num_beams=4,\n",
        "                                    no_repeat_ngram_size=2,\n",
        "                                    min_length=30,\n",
        "                                    max_length=100,\n",
        "                                    early_stopping=True)"
      ],
      "metadata": {
        "id": "myyvQFF3M2U0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "output7 = tokenizer.decode(summary7[0], skip_special_tokens=True)\n"
      ],
      "metadata": {
        "id": "xqFS2O4JdK75"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print (\"\\n\\nSummarized text: \\n\",output7)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q7p4-18odRcP",
        "outputId": "f3897186-7fa0-4c0a-987e-45363c8f28d4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "Summarized text: \n",
            " fake news can be defined as a pseudo-news or junk news. it is mainly created to mislead people or to cause ravage to someone, organization, etc., or gain financial benefits. the rise in internet network has changed the certain aspect of publishing and sharing the news with less editorial standards and regulations.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "gbO4deyadRfo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "3EBM21yUdK-4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "reference7 =\"Parameters including websites data, website validates, similarity of content, timestamps, reviews, and grammarical analysis calculate the authenticity of the news using the Naive Bayes and TFIDF algorithms that increase accuracy of the system.\"\n",
        "rouge = Rouge()\n",
        "rouge.get_scores(output3, reference7)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0oEYn3DnM2Yq",
        "outputId": "e68ab46d-5f0d-4c61-8c76-9fa3c973e09c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'rouge-1': {'r': 0.07407407407407407, 'p': 0.08, 'f': 0.0769230719304737},\n",
              "  'rouge-2': {'r': 0.0, 'p': 0.0, 'f': 0.0},\n",
              "  'rouge-l': {'r': 0.07407407407407407, 'p': 0.08, 'f': 0.0769230719304737}}]"
            ]
          },
          "metadata": {},
          "execution_count": 81
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Sample 8"
      ],
      "metadata": {
        "id": "T65NKlgfMVsl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#Sample 8: \n",
        "\n",
        "text8='''\n",
        "During the primeval eras, there were restricted choices and ways for personal iden- tification. \n",
        "Nowadays, we have an era of computer vision and biometrics, which does not involve any external artifact or token to recognize others. \n",
        "Instead, individuals can be acknowledged with their own biological or behavioral features with the aid of biometrics as an alternative to their associations, possessions, or any secret information.\n",
        "The necessity of mechanized and precise identification directed us to biometrics, which controls technology to accelerate the course of human identification and authentication. \n",
        "The biometric ID has been originated and replaced the printed IDs. This allows you to verify your identity, deprived of carrying any card or document (www.bayometric.com (accessed on 4 August 2021)). \n",
        "The authentication is a vital stage for offering admittance to the resources to the approved individuals. \n",
        "Conventional authentication systems such as a pin, card, and password cannot differentiate between real users and imposters who have fraudulently accessed the system.\n",
        " There are many possibilities of forgetting the password/pin or stealing and misplacement of the card. The device that allows the automatic identification of an individual is known as a biometric system. \n",
        " The biometric authentication system is easy to use, and there is no need to remember a password, card, and pin code.\n",
        "Biometrics have been extensively discovered for their automation, approachability, and accuracy with the mounting security needs of our everyday lives. \n",
        "It is a mechanized device that studies human beings’ physiological and behavioral features for their unique classification as the technology has differentiated from detection to criminal identifications and forensics.\n",
        "There are several diverse markets of biometric technology which exist today. Most of the markets appear to be mounting swiftly. \n",
        "The global biometric technologies market is anticipated to reach 19.08 billion US dollars in 2021, while the contactless biometric technologies market is predicted to grow to over 30.15 billion US dollars by 2027 (www. statista.com (accessed on 4 August 2021)).\n",
        " Biometrics have been successfully deployed in a variety of applications where security is of primary concern. \n",
        " \n",
        " For example, airport check-in and check-out personal identification cards [3]; sensitive information from unauthorized users, and credit card authentication.\n",
        "Several biometric characteristics such as the fingerprint, iris, palm print, and the face, are used for authentication and recognition.\n",
        " As compared to fingerprint and face, the iris-based authentication provides stronger contactless identification of the user. \n",
        " Table 1 displays various applications domains for iris biometric detection. The contactless approach helps to prevent the spread of viruses and diseases such as COVID-19. \n",
        " Iris has complex textures and unique features, so it is widely used in identifying and authentication the person in many applications. \n",
        " Aadhar project uses the biometric system to identify the citizens of India, Amsterdam airport, and the US Canadian border. Even though the iris has a unique texture pattern, there is a possibility of spoofing by the imposter. People attack the biometric device to obtain the rights of others.\n",
        "Iris detection systems can be easily spoofed by using different types of contact lens such as transparent lenses, colored lenses, and textured lenses.\n",
        " By using the transparent lenses, the fraudster cannot alter the iris texture but can modify the reflection property of the Iris recognition system. \n",
        " An imposter can conceal the real texture of an Iris with the aid of textured color lenses. The system can also be rapidly spoofed by replaying a video as well as a print attack. \n",
        " This means the iris pattern is acquainted with the machine by printing an iris image. Print attacks are performed in two modes:. \n",
        " First is print and scan, in which the high-quality printed iris pattern is scanned, and second is print and capture, in which the scanner takes the snapshot. \n",
        " Due to vulnerabilities in traditional security systems that lead to frequent security breaches, biometrics is increasingly becoming important.\n",
        "  Thus, ILD for biometric authentication is a significant research area.\n",
        "'''"
      ],
      "metadata": {
        "id": "kKv1V9EiMWhk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "preprocess_text8 = text8.strip().replace(\"\\n\",\"\")\n"
      ],
      "metadata": {
        "id": "ukuDZRgtM6w0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "t5_prepared_Text8 = \"summarize: \"+preprocess_text8\n"
      ],
      "metadata": {
        "id": "0wzzKzhPM6zi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_text8 = tokenizer.encode(t5_prepared_Text8, return_tensors=\"pt\")\n"
      ],
      "metadata": {
        "id": "Ex_y-3bBfaoD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "33e0ac5f-535d-4edc-8c71-94a4211a3d4c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Token indices sequence length is longer than the specified maximum sequence length for this model (870 > 512). Running this sequence through the model will result in indexing errors\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "summary8 = model.generate(tokenized_text8,\n",
        "                                    num_beams=4,\n",
        "                                    no_repeat_ngram_size=2,\n",
        "                                    min_length=30,\n",
        "                                    max_length=100,\n",
        "                                    early_stopping=True)"
      ],
      "metadata": {
        "id": "_R2EXOrUM65p"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "output8 = tokenizer.decode(summary8[0], skip_special_tokens=True)\n"
      ],
      "metadata": {
        "id": "c_ZreNsjdLr0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print (\"\\n\\nSummarized text: \\n\",output8)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cCtwoiVndSd0",
        "outputId": "fef6817a-2a36-4c36-9a18-02e923efbe9d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "Summarized text: \n",
            " biometrics is a mechanized device that studies human beings’ physiological and behavioral features for their unique classification as the technology has differentiated from detection to criminal identifications and forensics. it is easy to use, and there is no need to remember the password/pin or stealing and misplacement of the card.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "aXjLq61udSgd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "2_62WxfMdLue"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "reference8 =\"Biometrics is a mechanized device that studies human beings' physiological and behavioral features for their unique classification as the technology has differentiated from detection to criminal identifications and forensics. The biometric authentication system is easy to use, and there is no need to remember a password, card, and pin code.\"\n",
        "rouge.get_scores(output3, reference8)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YeEeZ-23M69L",
        "outputId": "e2aaf6c3-2887-42d6-a8ed-2880df5757bb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'rouge-1': {'r': 0.09523809523809523, 'p': 0.16, 'f': 0.11940298039652501},\n",
              "  'rouge-2': {'r': 0.0, 'p': 0.0, 'f': 0.0},\n",
              "  'rouge-l': {'r': 0.09523809523809523, 'p': 0.16, 'f': 0.11940298039652501}}]"
            ]
          },
          "metadata": {},
          "execution_count": 89
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "DOuMYO60MX-z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pickle"
      ],
      "metadata": {
        "id": "_hLu71Y6NzFR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "filename='t5.sav'\n",
        "pickle.dump(model, open(filename, 'wb'))"
      ],
      "metadata": {
        "id": "2ez7Sm4tN0W7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "filee='tokenizer.sav'\n",
        "pickle.dump(tokenizer, open(filee, 'wb'))"
      ],
      "metadata": {
        "id": "fGLaMqATPCYX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "yH4INTRUPdSB",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 346
        },
        "outputId": "2887e0ab-08af-4b59-fba4-d0810afb88d3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "MessageError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mMessageError\u001b[0m                              Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-92-d5df0069828e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mgoogle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolab\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdrive\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mdrive\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmount\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/content/drive'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/google/colab/drive.py\u001b[0m in \u001b[0;36mmount\u001b[0;34m(mountpoint, force_remount, timeout_ms, readonly)\u001b[0m\n\u001b[1;32m    104\u001b[0m       \u001b[0mtimeout_ms\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout_ms\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m       \u001b[0mephemeral\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 106\u001b[0;31m       readonly=readonly)\n\u001b[0m\u001b[1;32m    107\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    108\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/google/colab/drive.py\u001b[0m in \u001b[0;36m_mount\u001b[0;34m(mountpoint, force_remount, timeout_ms, ephemeral, readonly)\u001b[0m\n\u001b[1;32m    123\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0mephemeral\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    124\u001b[0m     _message.blocking_request(\n\u001b[0;32m--> 125\u001b[0;31m         'request_auth', request={'authType': 'dfs_ephemeral'}, timeout_sec=None)\n\u001b[0m\u001b[1;32m    126\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    127\u001b[0m   \u001b[0mmountpoint\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_os\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpanduser\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmountpoint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/google/colab/_message.py\u001b[0m in \u001b[0;36mblocking_request\u001b[0;34m(request_type, request, timeout_sec, parent)\u001b[0m\n\u001b[1;32m    169\u001b[0m   request_id = send_request(\n\u001b[1;32m    170\u001b[0m       request_type, request, parent=parent, expect_reply=True)\n\u001b[0;32m--> 171\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mread_reply_from_input\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout_sec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/google/colab/_message.py\u001b[0m in \u001b[0;36mread_reply_from_input\u001b[0;34m(message_id, timeout_sec)\u001b[0m\n\u001b[1;32m    100\u001b[0m         reply.get('colab_msg_id') == message_id):\n\u001b[1;32m    101\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0;34m'error'\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mreply\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 102\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mMessageError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreply\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'error'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    103\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mreply\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'data'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    104\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mMessageError\u001b[0m: Error: credential propagation was unsuccessful"
          ]
        }
      ]
    }
  ]
}