step5_publish_on_huggingface.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "source": "!pip install huggingface huggingface-cli huggingface_hub torch torchvision torchaudio transformers",
   "metadata": {
    "collapsed": true
   },
   "id": "initial_id",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "source": [
    "import os\n",
    "\n",
    "from typing import List, Dict, Union\n",
    "from tqdm import tqdm\n",
    "from transformers import AutoModelForTokenClassification, AutoTokenizer, BertForTokenClassification, BertTokenizerFast, pipeline, Pipeline\n",
    "\n",
    "from kediff_ner_system import KediffNerSystem"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "46bf6868db74093d",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "source": [
    "# !huggingface-cli login # do this in the terminal"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "b0badea1ae50d594",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "source": [
    "DATA_DIR: str\n",
    "try:\n",
    "    from google.colab import drive\n",
    "\n",
    "    print(\n",
    "        \"You work on Colab. Gentle as we are, we will mount Drive for you. \"\n",
    "        \"It'd help if you allowed this in the popup that opens.\"\n",
    "    )\n",
    "    drive.mount('/content/drive')\n",
    "    DATA_DIR = os.path.join('drive', 'MyDrive', 'KEDiff', 'data')\n",
    "except ModuleNotFoundError:\n",
    "    print(\"You do not work on Colab\")\n",
    "    DATA_DIR = os.path.join('data')\n",
    "print(f\"{DATA_DIR=}\", '-->', os.path.abspath(DATA_DIR))"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "a974200f4ab9bdd3",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "source": [
    "TOKENISER_CHECKPOINT: str = \"dbmdz/bert-base-historic-multilingual-cased\"\n",
    "CHECKPOINT_NAME_BASE: str = \"oalz-1788-q1-ner-\"\n",
    "TRAINED_DIR: str = os.path.join(DATA_DIR, 'trained_models', '2024-01-15')"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "5ef9ad66d875e373",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "source": "tokeniser: BertTokenizerFast = AutoTokenizer.from_pretrained(TOKENISER_CHECKPOINT)",
   "metadata": {
    "collapsed": false
   },
   "id": "ba4462c21f5d99a3",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "source": [
    "label_types: list[str] = [\"EVENT\", \"LOC\", \"MISC\", \"ORG\", \"PER\", \"TIME\"]\n",
    "selected_epochs: dict[str, str] = {\n",
    "    \"EVENT\": \"checkpoint-1393\",\n",
    "    \"LOC\": \"checkpoint-1393\",\n",
    "    \"MISC\": \"checkpoint-2786\",\n",
    "    \"ORG\": \"checkpoint-1393\",\n",
    "    \"PER\": \"checkpoint-2786\",\n",
    "    \"TIME\": \"checkpoint-1393\"\n",
    "}\n",
    "ner_model_paths: dict[str, str] = {\n",
    "    label_type: os.path.join(TRAINED_DIR,\n",
    "                             \"\".join([CHECKPOINT_NAME_BASE, label_type]),\n",
    "                             selected_epochs[label_type])\n",
    "    for label_type in label_types\n",
    "}\n",
    "ner_model_paths"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "70c78db4e8ba8e40",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "source": [
    "token_classifiers: dict[str, Pipeline] = {}\n",
    "label_type: str\n",
    "for label_type in tqdm(label_types):\n",
    "    token_classifiers[label_type] = pipeline(\n",
    "        \"token-classification\",\n",
    "        model=os.path.abspath(ner_model_paths[label_type]),\n",
    "        aggregation_strategy=\"simple\"\n",
    "    )\n",
    "list(token_classifiers.keys())"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "55b1f7d68a37a20f",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "source": [
    "label_type: str\n",
    "for label_type in label_types:\n",
    "    pipeline: Pipeline = token_classifiers[label_type]\n",
    "    tokeniser: BertTokenizerFast = pipeline.tokenizer\n",
    "    model: BertForTokenClassification = pipeline.model\n",
    "    model_name: str = f\"OALZ-1788-Q1-NER-{label_type}\"\n",
    "\n",
    "    # tokeniser.push_to_hub(repo_id=model_name)\n",
    "    # model.push_to_hub(repo_id=model_name)  # todo uncomment to actually publish"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "545cbc65c985bd0b",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "source": "sample_text: str = \"Dieses Modell soll nur eine der folgenden Kategorien erkennen: Personen-, Orts-, Organisations-, Ereignis-, Artefaktnamen oder Datumsangaben. Daher sprechen wir nun über Wilhelm Thell, der am 1. Jänner 1788, also noch vor der französischen Revolution und vor Vorstellung des ersten iPhones durch Apple im Jahre 2008, in Salzburg geboren wurde. Er studierte Pharmakologie an der Medizinischen Universität Wien.\"",
   "metadata": {
    "collapsed": false
   },
   "id": "6b10a272601e583e",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "source": [
    "person_model_hf: BertForTokenClassification = AutoModelForTokenClassification.from_pretrained(\"LelViLamp/OALZ-1788-Q1-NER-PER\")\n",
    "person_tokeniser: BertTokenizerFast = tokeniser"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "cf5bb6be0785698f",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "source": [
    "found_entities: list[dict[str, Union[float, int, str]]] = pipeline(\"ner\", model=person_model_hf, tokenizer=person_tokeniser)(sample_text)\n",
    "found_entities"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "38dbccd3dce08619",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "source": "KediffNerSystem.print_entities_as_table(found_entities)",
   "metadata": {
    "collapsed": false
   },
   "id": "90f139118a62ee0c",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "code",
   "source": "",
   "id": "fdf0dffbe16953c5",
   "outputs": [],
   "execution_count": null
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}