process-a-specific-file.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I need to process the /home/tajak/Parlamint-translation/Source-data/ParlaMint-CZ.conllu/ParlaMint-CZ.conllu/2016/ParlaMint-CZ_2016-10-26-ps2013-050-06-005-217.conllu in ParlaMint-CZ, because the file was empty when the previous version of the corpus was downloaded."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: CUDA_DEVICE_ORDER=PCI_BUS_ID\n",
      "env: CUDA_VISIBLE_DEVICES=5\n"
     ]
    }
   ],
   "source": [
    "%env CUDA_DEVICE_ORDER=PCI_BUS_ID\n",
    "%env CUDA_VISIBLE_DEVICES=5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import argparse\n",
    "import zipfile\n",
    "import gzip\n",
    "import shutil\n",
    "import pandas as pd"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "define main information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the language code, used in the file names\n",
    "#lang_code = \"CZ\"\n",
    "lang_code = \"SI\"\n",
    "\n",
    "# Main path\n",
    "main_path = \"/home/tajak/Parlamint-translation\"\n",
    "\n",
    "parl_list = [\"/home/tajak/Parlamint-translation/Source-data/ParlaMint-SI.conllu/ParlaMint-SI.conllu/2010/ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu\"]\n",
    "\n",
    "file_name_list = [\"ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu\"]\n",
    "\n",
    "extracted_dataframe_path = \"results/SI/ParlaMint-SI-extracted-source-data-missing-file.csv\"\n",
    "\n",
    "translated_dataframe_path = \"results/SI/ParlaMint-SI-translated-missing-file.csv\"\n",
    "\n",
    "translated_tokenized_dataframe_path = \"results/SI/ParlaMint-SI-translated-tokenized-missing-file.csv\"\n",
    "\n",
    "final_dataframe = \"results/SI/ParlaMint-SI-final-dataframe-missing-file.csv\""
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "conllu_to-df()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of words in the corpora: 55184\n",
      "Dataframe saved as results/SI/ParlaMint-SI-extracted-source-data-missing-file.csv\n",
      "|        | file_path                                                                                                                                 | file                                           | sentence_id                                        | text        | tokenized_text   | proper_nouns   |    length |\n",
      "|:-------|:------------------------------------------------------------------------------------------------------------------------------------------|:-----------------------------------------------|:---------------------------------------------------|:------------|:-----------------|:---------------|----------:|\n",
      "| count  | 3042                                                                                                                                      | 3042                                           | 3042                                               | 3042        | 3042             | 3042           | 3042      |\n",
      "| unique | 1                                                                                                                                         | 1                                              | 3042                                               | 2668        | 2668             | 196            |  nan      |\n",
      "| top    | /home/tajak/Parlamint-translation/Source-data/ParlaMint-SI.conllu/ParlaMint-SI.conllu/2010/ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.ana.seg1.1 | Hvala lepa. | Hvala lepa .     | {}             |  nan      |\n",
      "| freq   | 3042                                                                                                                                      | 3042                                           | 1                                                  | 135         | 135              | 2810           |  nan      |\n",
      "| mean   | nan                                                                                                                                       | nan                                            | nan                                                | nan         | nan              | nan            |   18.1407 |\n",
      "| std    | nan                                                                                                                                       | nan                                            | nan                                                | nan         | nan              | nan            |   15.2758 |\n",
      "| min    | nan                                                                                                                                       | nan                                            | nan                                                | nan         | nan              | nan            |    1      |\n",
      "| 25%    | nan                                                                                                                                       | nan                                            | nan                                                | nan         | nan              | nan            |    6      |\n",
      "| 50%    | nan                                                                                                                                       | nan                                            | nan                                                | nan         | nan              | nan            |   14      |\n",
      "| 75%    | nan                                                                                                                                       | nan                                            | nan                                                | nan         | nan              | nan            |   27      |\n",
      "| max    | nan                                                                                                                                       | nan                                            | nan                                                | nan         | nan              | nan            |  148      |\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "|    | file_path                                                                                                                                 | file                                           | sentence_id                                        | text                                                                                                                                                                                                                                                                          | tokenized_text                                                                                                                                                                                                                                                                       | proper_nouns                                                                                                                                                                                                                                                                                                                                         |   length |\n",
      "|---:|:------------------------------------------------------------------------------------------------------------------------------------------|:-----------------------------------------------|:---------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------:|\n",
      "|  0 | /home/tajak/Parlamint-translation/Source-data/ParlaMint-SI.conllu/ParlaMint-SI.conllu/2010/ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.ana.seg1.1 | Spoštovane kolegice poslanke in kolegi poslanci, gospe in gospodje!                                                                                                                                                                                                           | Spoštovane kolegice poslanke in kolegi poslanci , gospe in gospodje !                                                                                                                                                                                                                | {}                                                                                                                                                                                                                                                                                                                                                   |        9 |\n",
      "|  1 | /home/tajak/Parlamint-translation/Source-data/ParlaMint-SI.conllu/ParlaMint-SI.conllu/2010/ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.ana.seg2.1 | Začenjam 34. izredno sejo Državnega zbora, ki sem jo sklical na podlagi prvega odstavka 58. člena ter drugega odstavka 60. člena Poslovnika Državnega zbora.                                                                                                                  | Začenjam 34. izredno sejo Državnega zbora , ki sem jo sklical na podlagi prvega odstavka 58. člena ter drugega odstavka 60. člena Poslovnika Državnega zbora .                                                                                                                       | {}                                                                                                                                                                                                                                                                                                                                                   |       24 |\n",
      "|  2 | /home/tajak/Parlamint-translation/Source-data/ParlaMint-SI.conllu/ParlaMint-SI.conllu/2010/ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.ana.seg3.1 | Obveščen sem, da se današnje seje ne morejo udeležiti naslednja poslanka in poslanci: Eva Irgl, Zmago Jelinčič Plemeniti, Rudolf Petan, Roberto Battelli in tukaj imam napisano dr. Vinko Gorenak, ki pa je tukaj – od 16. ure dalje se Vinko Gorenak ne more udeležiti seje. | Obveščen sem , da se današnje seje ne morejo udeležiti naslednja poslanka in poslanci : Eva Irgl , Zmago Jelinčič Plemeniti , Rudolf Petan , Roberto Battelli in tukaj imam napisano dr. Vinko Gorenak , ki pa je tukaj – od 16. ure dalje se Vinko Gorenak ne more udeležiti seje . | {15: ['Eva', 'Eva'], 16: ['Irgl', 'Irgl'], 18: ['Zmago', 'Zmago'], 19: ['Jelinčič', 'Jelinčič'], 20: ['Plemeniti', 'Plemeniti'], 22: ['Rudolf', 'Rudolf'], 23: ['Petan', 'Petan'], 25: ['Roberto', 'Roberto'], 26: ['Battelli', 'Battelli'], 32: ['Vinko', 'Vinko'], 33: ['Gorenak', 'Gorenak'], 45: ['Vinko', 'Vinko'], 46: ['Gorenak', 'Gorenak']} |       45 |\n",
      "|  3 | /home/tajak/Parlamint-translation/Source-data/ParlaMint-SI.conllu/ParlaMint-SI.conllu/2010/ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.ana.seg4.1 | Na sejo sem vabil predstavnike Državnega sveta k odločitvi dnevnega reda seje ter predstavnike Vlade k vsem točkam dnevnega reda.                                                                                                                                             | Na sejo sem vabil predstavnike Državnega sveta k odločitvi dnevnega reda seje ter predstavnike Vlade k vsem točkam dnevnega reda .                                                                                                                                                   | {}                                                                                                                                                                                                                                                                                                                                                   |       20 |\n",
      "|  4 | /home/tajak/Parlamint-translation/Source-data/ParlaMint-SI.conllu/ParlaMint-SI.conllu/2010/ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.ana.seg5.1 | Vse prisotne lepo pozdravljam.                                                                                                                                                                                                                                                | Vse prisotne lepo pozdravljam .                                                                                                                                                                                                                                                      | {}                                                                                                                                                                                                                                                                                                                                                   |        4 |\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def conllu_to_df(parl_list, file_name_list, extracted_dataframe_path):\n",
    "\t\"\"\"\n",
    "\tTake the conllu files and extract relevant information. Save everything in a DataFrame.\n",
    "\n",
    "\tArgs:\n",
    "\t- parl_list: list of documents with their entire paths to be included (see step above).\n",
    "\t- file_name_list: list of names of the files (see step above)\n",
    "\t- extracted_dataframe_path: path to the output file\n",
    "\t\"\"\"\n",
    "\tfrom conllu import parse\n",
    "\timport pandas as pd\n",
    "\n",
    "\t# Create an empty df\n",
    "\tdf = pd.DataFrame({\"file_path\": [\"\"],\"file\": [\"\"], \"sentence_id\": [\"\"], \"text\": [\"\"], \"tokenized_text\": [\"\"], \"proper_nouns\": [\"\"]})\n",
    "\n",
    "\t# Check whether there are any problems with parsing the documents\n",
    "\t\"\"\"\n",
    "\t\n",
    "\terror_count = 0\n",
    "\tproblematic_doc_list = []\n",
    "\n",
    "\tfor doc in parl_list:\n",
    "\t\ttry:\n",
    "\t\t\t# Open the file\n",
    "\t\t\tdata = open(\"{}\".format(doc), \"r\").read()\n",
    "\n",
    "\t\t\tsentences = parse(data)\n",
    "\t\texcept:\n",
    "\t\t\terror_count += 1\n",
    "\t\t\tproblematic_doc_list.append(doc)\n",
    "\n",
    "\tprint(error_count)\n",
    "\tprint(problematic_doc_list)\n",
    "\t\"\"\"\n",
    "\t# Parse the data with CONLL-u parser\n",
    "\tfor doc in parl_list:\n",
    "\t\t# Open the file\n",
    "\t\tdata = open(\"{}\".format(doc), \"r\").read()\n",
    "\t\t\n",
    "\t\tsentences = parse(data)\n",
    "\n",
    "\t\tsentence_id_list = []\n",
    "\t\ttext_list = []\n",
    "\t\ttokenized_text_list = []\n",
    "\t\tproper_noun_list = []\n",
    "\n",
    "\t\tfor sentence in sentences:\n",
    "\t\t\t# Find sentence ids\n",
    "\t\t\tcurrent_sentence_id = sentence.metadata[\"sent_id\"]\n",
    "\t\t\tsentence_id_list.append(current_sentence_id)\n",
    "\n",
    "\t\t\t# Find text - if texts consists of multiword tokens, these tokens will appear as they are,\n",
    "\t\t\t# not separated into subwords\n",
    "\t\t\tcurrent_text = sentence.metadata[\"text\"]\n",
    "\t\t\ttext_list.append(current_text)\n",
    "\n",
    "\t\t\t# Create a string out of tokens\n",
    "\t\t\tcurrent_token_list = []\n",
    "\t\t\tword_dict = {}\n",
    "\n",
    "\t\t\tfor token in sentence:\n",
    "\t\t\t\t# Find multiword tokens and take their NER\n",
    "\t\t\t\tif type(token[\"id\"]) != int:\n",
    "\t\t\t\t\tmultiword_ner = token[\"misc\"][\"NER\"]\n",
    "\t\t\t\t\n",
    "\t\t\t\telse:\n",
    "\t\t\t\t# Append to the tokenized text tokens that are not multiword tokens\n",
    "\t\t\t\t# (we append subtokens to the tokenized texts, not multiword tokens)\n",
    "\t\t\t\t\tcurrent_token_list.append(token[\"form\"])\n",
    "\t\t\t\t\t\n",
    "\n",
    "\t\t\t\t\t# Create a list of NE annotations with word indices.\n",
    "\t\t\t\t\t# I'll substract one from the word index,\n",
    "\t\t\t\t\t# because indexing in the CONLLU file starts with 1, not 0\n",
    "\t\t\t\t\tcurrent_index = int(token[\"id\"]) - 1\n",
    "\n",
    "\t\t\t\t\t# If the word does not have NER annotation,\n",
    "\t\t\t\t\t# take the annotation from the multiword token\n",
    "\t\t\t\t\tif token[\"misc\"] is None:\n",
    "\t\t\t\t\t\tcurrent_ner = multiword_ner\n",
    "\t\t\t\t\telse:\n",
    "\t\t\t\t\t\tcurrent_ner = token[\"misc\"][\"NER\"]\n",
    "\n",
    "\t\t\t\t\t# Add information on the lemma if the NE is personal name\n",
    "\t\t\t\t\tif \"PER\" in current_ner:\n",
    "\t\t\t\t\t\tword_dict[current_index] = [token[\"form\"], token[\"lemma\"]]\n",
    "\n",
    "\t\t\tproper_noun_list.append(word_dict)\n",
    "\n",
    "\t\t\tcurrent_string = \" \".join(current_token_list)\n",
    "\n",
    "\t\t\ttokenized_text_list.append(current_string)\n",
    "\n",
    "\t\t\n",
    "\t\tnew_df = pd.DataFrame({\"sentence_id\": sentence_id_list, \"text\": text_list, \"tokenized_text\": tokenized_text_list, \"proper_nouns\": proper_noun_list})\n",
    "\n",
    "\t\tnew_df[\"file_path\"] = doc\n",
    "\n",
    "\t\t# Get the file name\n",
    "\t\tfile_name = file_name_list[parl_list.index(doc)]\n",
    "\t\tnew_df[\"file\"] = file_name\n",
    "\n",
    "\t\t# Merge df to the previous df\n",
    "\t\tdf = pd.concat([df, new_df])\n",
    "\t\n",
    "\t# Reset index\n",
    "\tdf = df.reset_index(drop=True)\n",
    "\n",
    "\t# Remove the first row\n",
    "\tdf = df.drop([0], axis=\"index\")\n",
    "\n",
    "\t# Reset index\n",
    "\tdf = df.reset_index(drop=True)\n",
    "\n",
    "\t# Add information on length\n",
    "\tdf[\"length\"] = df[\"text\"].str.split().str.len()\n",
    "\n",
    "\tprint(\"Number of words in the corpora: {}\".format(df[\"length\"].sum()))\n",
    "\n",
    "\t# Save the dataframe\n",
    "\tdf.to_csv(\"{}\".format(extracted_dataframe_path), sep=\"\\t\")\n",
    "\n",
    "\tprint(\"Dataframe saved as {}\".format(extracted_dataframe_path))\n",
    "\t\n",
    "\t# Show the results\n",
    "\tprint(df.describe(include=\"all\").to_markdown())\n",
    "\n",
    "\tprint(\"\\n\\n\\n\")\n",
    "\n",
    "\tprint(df.head().to_markdown())\n",
    "\n",
    "\tprint(\"\\n\\n\\n\")\n",
    "\t\n",
    "\treturn df\n",
    "\n",
    "#Extract information from the conllu files\n",
    "df = conllu_to_df(parl_list, file_name_list, extracted_dataframe_path)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "translate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/tajak/Parlamint-translation/parlamint_env/lib/python3.8/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "2023-03-10 11:44:58.008238: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-03-10 11:44:58.722493: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory\n",
      "2023-03-10 11:44:58.722588: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory\n",
      "2023-03-10 11:44:58.722593: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Entire corpus has 3042 sentences and 55184 words.\n",
      "Translation started.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/tajak/Parlamint-translation/parlamint_env/lib/python3.8/site-packages/transformers/generation/utils.py:1273: UserWarning: Neither `max_length` nor `max_new_tokens` has been set, `max_length` will default to 512 (`generation_config.max_length`). Controlling `max_length` via the config is deprecated and `max_length` will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Translation completed. It took 1.39 minutes for 3042 instances - 0.00045693622616699536 minutes per one sentence.\n",
      "|    | file_path                                                                                                                                 | file                                           | sentence_id                                        | text                                                                                                                                                                                                                                                                          | tokenized_text                                                                                                                                                                                                                                                                       | proper_nouns                                                                                                                                                                                                                                                                                                                                         |   length | translation                                                                                                                                                                                                                                                                            |\n",
      "|---:|:------------------------------------------------------------------------------------------------------------------------------------------|:-----------------------------------------------|:---------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|\n",
      "|  0 | /home/tajak/Parlamint-translation/Source-data/ParlaMint-SI.conllu/ParlaMint-SI.conllu/2010/ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.ana.seg1.1 | Spoštovane kolegice poslanke in kolegi poslanci, gospe in gospodje!                                                                                                                                                                                                           | Spoštovane kolegice poslanke in kolegi poslanci , gospe in gospodje !                                                                                                                                                                                                                | {}                                                                                                                                                                                                                                                                                                                                                   |        9 | Dear fellow Members and fellow Members, ladies and gentlemen!                                                                                                                                                                                                                          |\n",
      "|  1 | /home/tajak/Parlamint-translation/Source-data/ParlaMint-SI.conllu/ParlaMint-SI.conllu/2010/ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.ana.seg2.1 | Začenjam 34. izredno sejo Državnega zbora, ki sem jo sklical na podlagi prvega odstavka 58. člena ter drugega odstavka 60. člena Poslovnika Državnega zbora.                                                                                                                  | Začenjam 34. izredno sejo Državnega zbora , ki sem jo sklical na podlagi prvega odstavka 58. člena ter drugega odstavka 60. člena Poslovnika Državnega zbora .                                                                                                                       | {}                                                                                                                                                                                                                                                                                                                                                   |       24 | I am opening the 34th extraordinary meeting of the National Assembly convened pursuant to Article 58, paragraph 1, and Article 60, paragraph 2, of the Rules of Procedure.                                                                                                             |\n",
      "|  2 | /home/tajak/Parlamint-translation/Source-data/ParlaMint-SI.conllu/ParlaMint-SI.conllu/2010/ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.ana.seg3.1 | Obveščen sem, da se današnje seje ne morejo udeležiti naslednja poslanka in poslanci: Eva Irgl, Zmago Jelinčič Plemeniti, Rudolf Petan, Roberto Battelli in tukaj imam napisano dr. Vinko Gorenak, ki pa je tukaj – od 16. ure dalje se Vinko Gorenak ne more udeležiti seje. | Obveščen sem , da se današnje seje ne morejo udeležiti naslednja poslanka in poslanci : Eva Irgl , Zmago Jelinčič Plemeniti , Rudolf Petan , Roberto Battelli in tukaj imam napisano dr. Vinko Gorenak , ki pa je tukaj – od 16. ure dalje se Vinko Gorenak ne more udeležiti seje . | {15: ['Eva', 'Eva'], 16: ['Irgl', 'Irgl'], 18: ['Zmago', 'Zmago'], 19: ['Jelinčič', 'Jelinčič'], 20: ['Plemeniti', 'Plemeniti'], 22: ['Rudolf', 'Rudolf'], 23: ['Petan', 'Petan'], 25: ['Roberto', 'Roberto'], 26: ['Battelli', 'Battelli'], 32: ['Vinko', 'Vinko'], 33: ['Gorenak', 'Gorenak'], 45: ['Vinko', 'Vinko'], 46: ['Gorenak', 'Gorenak']} |       45 | I am informed that today's meetings cannot be attended by the following Members and Members: Eva Irgl, Victory Jelinčič Trimeniti, Rudolf Petan, Roberto Battelli and here I have written Dr. Vinko Gorenak, who is here – from 16:00 onwards Vinko Gorenak cannot attend the meeting. |\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "The file is saved as results/SI/ParlaMint-SI-translated-missing-file.csv\n"
     ]
    }
   ],
   "source": [
    "def translate(lang_code, extracted_dataframe_path, translated_dataframe_path):\n",
    "\t\"\"\"\n",
    "\tThis function translates the text from the dataframe, created with the extract_text() function\n",
    "\twith OPUS-MT models using EasyNMT. It returns a dataframe with the translation.\n",
    "\n",
    "\tArgs:\n",
    "\t- opus_lang_code: the lang code to be used in the OPUS-MT model - use the one that performed the best in the comparison (see function choose_model())\n",
    "\t\"\"\"\n",
    "\timport pandas as pd\n",
    "\timport regex as re\n",
    "\tfrom easynmt import EasyNMT\n",
    "\tfrom IPython.display import display\n",
    "\timport time\n",
    "\n",
    "\t# Open the file, created in the previous step\n",
    "\tdf = pd.read_csv(\"{}\".format(extracted_dataframe_path), sep=\"\\t\", index_col=0)\n",
    "\n",
    "\t# Define the model\n",
    "\tmodel = EasyNMT('opus-mt')\n",
    "\n",
    "\tprint(\"Entire corpus has {} sentences and {} words.\".format(df[\"text\"].count(), df[\"length\"].sum()))\n",
    "\n",
    "\t# Create a list of sentences from the df\n",
    "\tsentence_list = df.text.to_list()\n",
    "\n",
    "\tlang_models_dict = {\"BG\": [\"bg\", \"sla\", \"zls\"], \"HR\": [\"zls\", \"sla\"], \"CZ\": [\"cs\"], \"DK\": [\"da\", \"gmq\", \"gem\"], \"NL\": [\"nl\", \"gem\", \"gmw\"], \"FR\": [\"fr\", \"itc\",\"roa\"], \"HU\": [\"hu\", \"fiu\", \"urj\"], \"IS\": [\"is\",\"gmq\", \"gem\"], \"IT\": [\"it\", \"roa\", \"itc\"], \"LV\": [\"lv\",\"bat\"], \"LT\": [\"bat\"], \"PL\": [\"pl\", \"sla\", \"zlw\"], \"SI\": [\"sla\"], \"ES\": [\"es\", \"roa\", \"itc\"], \"TR\": [\"tr\", \"trk\" ], \"AT\": [\"de\", \"gem\", \"gmw\"], \"ES-PV\": [\"eu\", \"mul\"], \"BA\": [\"sla\", \"zls\"], \"ES-CT\": [\"ca\", \"roa\", \"itc\"], \"EE\": [\"et\", \"urj\", \"fiu\"], \"FI\": [\"fi\", \"urj\", \"fiu\"], \"ES-GA\": [\"gl\", \"roa\", \"itc\"], \"GR\": [\"el\",\"grk\"], \"NO\": [\"gem\", \"gmq\"], \"PT\": [\"pt\", \"roa\", \"itc\"], \"RO\":[\"roa\", \"itc\"], \"RS\": [\"zls\", \"sla\"], \"SE\": [\"sv\", \"gmq\", \"gem\"], \"UA\":[\"uk\", \"sla\", \"zle\"]}\n",
    "\n",
    "\tprint(\"Translation started.\")\n",
    "\n",
    "\tstart_time = time.time()\n",
    "\n",
    "\t#Translate the list of sentences - you need to provide the source language as it is in the name of the model - the opus_lang_code\n",
    "\tfor opus_lang_code in lang_models_dict[lang_code]:\n",
    "\t\ttranslation_list = model.translate(sentence_list, source_lang = \"{}\".format(opus_lang_code), target_lang='en')\n",
    "\n",
    "\ttranslation_time = round((time.time() - start_time)/60,2)\n",
    "\n",
    "\tprint(\"Translation completed. It took {} minutes for {} instances - {} minutes per one sentence.\".format(translation_time, len(sentence_list), translation_time/len(sentence_list)))\n",
    "\n",
    "\t# Add the translations to the df\n",
    "\tdf[\"translation\"] = translation_list\n",
    "\n",
    "\t# Display the df\n",
    "\tprint(df[:3].to_markdown())\n",
    "\n",
    "\tprint(\"\\n\\n\\n\")\n",
    "\n",
    "\t# Save the df\n",
    "\tdf.to_csv(\"{}\".format(translated_dataframe_path), sep=\"\\t\")\n",
    "\n",
    "\tprint(\"The file is saved as {}\".format(translated_dataframe_path))\n",
    "\n",
    "\treturn df\n",
    "\n",
    "df = translate(lang_code, extracted_dataframe_path, translated_dataframe_path)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "prior file for alignment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/tajak/Parlamint-translation/eflomal\n"
     ]
    }
   ],
   "source": [
    "# Create a prior file from alignments, created on the big file, to align the sentences based on them\n",
    "\n",
    "# move into the eflomal directory\n",
    "%cd eflomal\n",
    "\n",
    "# First, we need to create a fasttext format from the two sentence files\n",
    "!python3 mergefiles.py \"source_sentences_SI.txt\" \"English_sentences_SI.txt\" > si-en.file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now, create the prior file\n",
    "!python3 makepriors.py -i si-en.file -f source-en_SI.fwd -r source-en_SI.rev --priors si-en.priors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now, if you have another file to align, simply use the original sentences and the prior file:\n",
    "#!python3 align.py -s \"source_sentences_CZ.txt\" -t \"English_sentences_CZ.txt\" --priors cz-en.priors --model 3 -f source-en-CZ-add-file.fwd -r source-en-CZ-add-file.rev"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "tokenize_translation()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-10 11:56:11 INFO: Loading these models for language: en (English):\n",
      "========================\n",
      "| Processor | Package  |\n",
      "------------------------\n",
      "| tokenize  | combined |\n",
      "========================\n",
      "\n",
      "2023-03-10 11:56:11 INFO: Use device: gpu\n",
      "2023-03-10 11:56:11 INFO: Loading: tokenize\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/tajak/Parlamint-translation\n",
      "Tokenization of the translation started.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-10 11:56:11 INFO: Done loading processors!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tokenization completed. It took 0.1 minutes.\n",
      "File saved as results/SI/ParlaMint-SI-translated-tokenized-missing-file.csv\n",
      "|    | file_path                                                                                                                                 | file                                           | sentence_id                                        | text                                                                                                                                                                                                                                                                          | tokenized_text                                                                                                                                                                                                                                                                       | proper_nouns                                                                                                                                                                                                                                                                                                                                         |   length | translation                                                                                                                                                                                                                                                                            | translation-tokenized                                                                                                                                                                                                                                                                         | space-after-information                                                                                                                                                                                                                                                                                                                                                |\n",
      "|---:|:------------------------------------------------------------------------------------------------------------------------------------------|:-----------------------------------------------|:---------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|\n",
      "|  0 | /home/tajak/Parlamint-translation/Source-data/ParlaMint-SI.conllu/ParlaMint-SI.conllu/2010/ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.ana.seg1.1 | Spoštovane kolegice poslanke in kolegi poslanci, gospe in gospodje!                                                                                                                                                                                                           | Spoštovane kolegice poslanke in kolegi poslanci , gospe in gospodje !                                                                                                                                                                                                                | {}                                                                                                                                                                                                                                                                                                                                                   |        9 | Dear fellow Members and fellow Members, ladies and gentlemen!                                                                                                                                                                                                                          | Dear fellow Members and fellow Members , ladies and gentlemen !                                                                                                                                                                                                                               | ['Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'No', 'Yes', 'Yes', 'Yes', 'No', 'Last']                                                                                                                                                                                                                                                                                           |\n",
      "|  1 | /home/tajak/Parlamint-translation/Source-data/ParlaMint-SI.conllu/ParlaMint-SI.conllu/2010/ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.ana.seg2.1 | Začenjam 34. izredno sejo Državnega zbora, ki sem jo sklical na podlagi prvega odstavka 58. člena ter drugega odstavka 60. člena Poslovnika Državnega zbora.                                                                                                                  | Začenjam 34. izredno sejo Državnega zbora , ki sem jo sklical na podlagi prvega odstavka 58. člena ter drugega odstavka 60. člena Poslovnika Državnega zbora .                                                                                                                       | {}                                                                                                                                                                                                                                                                                                                                                   |       24 | I am opening the 34th extraordinary meeting of the National Assembly convened pursuant to Article 58, paragraph 1, and Article 60, paragraph 2, of the Rules of Procedure.                                                                                                             | I am opening the 34th extraordinary meeting of the National Assembly convened pursuant to Article 58 , paragraph 1 , and Article 60 , paragraph 2 , of the Rules of Procedure .                                                                                                               | ['Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'No', 'Yes', 'Yes', 'No', 'Yes', 'Yes', 'Yes', 'No', 'Yes', 'Yes', 'No', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'No', 'Last']                                                                                                                                    |\n",
      "|  2 | /home/tajak/Parlamint-translation/Source-data/ParlaMint-SI.conllu/ParlaMint-SI.conllu/2010/ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.ana.seg3.1 | Obveščen sem, da se današnje seje ne morejo udeležiti naslednja poslanka in poslanci: Eva Irgl, Zmago Jelinčič Plemeniti, Rudolf Petan, Roberto Battelli in tukaj imam napisano dr. Vinko Gorenak, ki pa je tukaj – od 16. ure dalje se Vinko Gorenak ne more udeležiti seje. | Obveščen sem , da se današnje seje ne morejo udeležiti naslednja poslanka in poslanci : Eva Irgl , Zmago Jelinčič Plemeniti , Rudolf Petan , Roberto Battelli in tukaj imam napisano dr. Vinko Gorenak , ki pa je tukaj – od 16. ure dalje se Vinko Gorenak ne more udeležiti seje . | {15: ['Eva', 'Eva'], 16: ['Irgl', 'Irgl'], 18: ['Zmago', 'Zmago'], 19: ['Jelinčič', 'Jelinčič'], 20: ['Plemeniti', 'Plemeniti'], 22: ['Rudolf', 'Rudolf'], 23: ['Petan', 'Petan'], 25: ['Roberto', 'Roberto'], 26: ['Battelli', 'Battelli'], 32: ['Vinko', 'Vinko'], 33: ['Gorenak', 'Gorenak'], 45: ['Vinko', 'Vinko'], 46: ['Gorenak', 'Gorenak']} |       45 | I am informed that today's meetings cannot be attended by the following Members and Members: Eva Irgl, Victory Jelinčič Trimeniti, Rudolf Petan, Roberto Battelli and here I have written Dr. Vinko Gorenak, who is here – from 16:00 onwards Vinko Gorenak cannot attend the meeting. | I am informed that today 's meetings cannot be attended by the following Members and Members : Eva Irgl , Victory Jelinčič Trimeniti , Rudolf Petan , Roberto Battelli and here I have written Dr. Vinko Gorenak , who is here – from 16:00 onwards Vinko Gorenak cannot attend the meeting . | ['Yes', 'Yes', 'Yes', 'Yes', 'No', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'No', 'Yes', 'Yes', 'No', 'Yes', 'Yes', 'Yes', 'No', 'Yes', 'Yes', 'No', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'No', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'No', 'Last'] |\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%cd ..\n",
    "\n",
    "def tokenize_translation(translated_dataframe_path, translated_tokenized_dataframe_path):\n",
    "\timport stanza\n",
    "\timport time\n",
    "\timport gc\n",
    "\timport torch\n",
    "\tfrom stanza.pipeline.core import DownloadMethod\n",
    "\t\n",
    "\tprint(\"Tokenization of the translation started.\")\n",
    "\n",
    "\tnlp = stanza.Pipeline(lang='en', processors='tokenize', tokenize_no_ssplit = True, download_method=DownloadMethod.REUSE_RESOURCES, use_gpu=True)\n",
    "\n",
    "\t# Apply tokenization to English translation and add the sentences to the df\n",
    "\t# Open the df\n",
    "\tdf = pd.read_csv(\"{}\".format(translated_dataframe_path), sep=\"\\t\", index_col = 0)\n",
    "\n",
    "\t# Save also the information on whether there is a space after or before punctuation\n",
    "\t# which we will need later, to remove unnecessary spaces\n",
    "\tEn_sentences = df.translation.to_list()\n",
    "\n",
    "\ttokenized_sentences = []\n",
    "\tspace_after_list = []\n",
    "\n",
    "\tstart_time = time.time()\n",
    "\n",
    "\tfor i in En_sentences:\n",
    "\t\tdoc = nlp(i).to_dict()\n",
    "\t\tcurrent_sentence_list = []\n",
    "\t\tcurrent_space_after_list = []\n",
    "\n",
    "\t\t# Define a list of start_char and end_char\n",
    "\t\tstart_chars = []\n",
    "\t\tend_chars = []\n",
    "\n",
    "\t\t# Loop through the tokens in the sentence and add them to a current sentence list\n",
    "\t\tfor sentence in doc:\n",
    "\t\t\tfor word in sentence:\n",
    "\t\t\t\tcurrent_sentence_list.append(word[\"text\"])\n",
    "\n",
    "\t\t\t\t# Add information on start and end chars to the list\n",
    "\t\t\t\tstart_chars.append(word[\"start_char\"])\n",
    "\t\t\t\tend_chars.append(word[\"end_char\"])\n",
    "\t\t\t\n",
    "\t\t# Now loop through the start_char and end_char lists and find instances\n",
    "\t\t# where the end_char of one word is the same as the start_char of the next one\n",
    "\t\t# this means there is no space between them\n",
    "\t\tfor char_index in range(len(start_chars)-1):\n",
    "\t\t\tif end_chars[char_index] == start_chars[(char_index+1)]:\n",
    "\t\t\t\tcurrent_space_after_list.append(\"No\")\n",
    "\t\t\telse:\n",
    "\t\t\t\tcurrent_space_after_list.append(\"Yes\")\n",
    "\t\n",
    "\t\t# This loop is not possible for the end token, so let's add information for the last token\n",
    "\t\t# just to avoid errors due to different lengths of lists\n",
    "\t\tcurrent_space_after_list.append(\"Last\")\n",
    "\n",
    "\t\t# Join the list into a space-separated string\n",
    "\t\tcurrent_string = \" \".join(current_sentence_list)\n",
    "\n",
    "\t\ttokenized_sentences.append(current_string)\n",
    "\n",
    "\t\tspace_after_list.append(current_space_after_list)\n",
    "\n",
    "\t# Add the result to the df\n",
    "\tdf[\"translation-tokenized\"] = tokenized_sentences\n",
    "\tdf[\"space-after-information\"] = space_after_list\n",
    "\n",
    "\t# Save the df\n",
    "\tdf.to_csv(\"{}\".format(translated_tokenized_dataframe_path), sep=\"\\t\")\n",
    "\n",
    "\tend_time = round((time.time() - start_time)/60,2)\n",
    "\n",
    "\tprint(\"Tokenization completed. It took {} minutes.\".format(end_time))\n",
    "\n",
    "\t#Delete nlp element, clean memory\n",
    "\tdel nlp\n",
    "\ttorch.cuda.empty_cache()\n",
    "\tgc.collect()\n",
    "\ttorch.cuda.empty_cache()\n",
    "\n",
    "\tprint(\"File saved as {}\".format(translated_tokenized_dataframe_path))\n",
    "\t\n",
    "\treturn df\n",
    "\n",
    "df = tokenize_translation(translated_dataframe_path, translated_tokenized_dataframe_path)\n",
    "\n",
    "print(df.head(3).to_markdown())\n",
    "print(\"\\n\\n\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "align - change the alignment files accordingly to your language in the code (after \"# Align sentences with eflomal and get out a file with alignments\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "Alignment started.\n",
      "\n",
      "Number of aligned sentences: 3042\n",
      "\n",
      "\n",
      "Alignment completed. It took 0.21 minutes.\n",
      "|                                                    |   substituted_pairs |\n",
      "|:---------------------------------------------------|--------------------:|\n",
      "| 0                                                  |                2903 |\n",
      "| [('Cuckoo', 'Cukjati')]                            |                  10 |\n",
      "| [('Franz', 'Franc')]                               |                   6 |\n",
      "| [('Rozej', 'Rožej')]                               |                   5 |\n",
      "| [('President', 'Presečnik')]                       |                   5 |\n",
      "| [('Tsukati', 'Cukjati')]                           |                   4 |\n",
      "| [('Joseph', 'Jožef')]                              |                   4 |\n",
      "| [('Kampush', 'Kampuš')]                            |                   4 |\n",
      "| [('Germić', 'Germič')]                             |                   3 |\n",
      "| [('Franz', 'Franc'), ('Gniedarsich', 'Žnidaršič')] |                   3 |\n",
      "| [('Danil', 'Danilo')]                              |                   3 |\n",
      "| [('Visjak', 'Vizjak')]                             |                   3 |\n",
      "| [('Pecan', 'Pečan')]                               |                   2 |\n",
      "| [('Sayovic', 'Sajovic')]                           |                   2 |\n",
      "| [('Marian', 'Marijan'), ('Pojbich', 'Pojbič')]     |                   2 |\n",
      "| [('visor', 'Vizjak')]                              |                   2 |\n",
      "| [('Luke', 'Luka')]                                 |                   2 |\n",
      "| [('Barovich', 'Barovič')]                          |                   2 |\n",
      "| [('Black', 'Črnak'), ('Neglec', 'Meglič')]         |                   2 |\n",
      "| [('Grimes', 'Grims')]                              |                   2 |\n",
      "\n",
      "\n",
      "\n",
      "Number of errors:\n",
      "(0, 19)\n",
      "\n",
      "\n",
      "\n",
      "Example of sentences with substituted words.\n",
      "|    | file_path                                                                                                                                 | file                                           | sentence_id                                        | text                                                                                                                                                                                                                                                                          | tokenized_text                                                                                                                                                                                                                                                                       | proper_nouns                                                                                                                                                                                                                                                                                                                                         |   length | translation                                                                                                                                                                                                                                                                                            | translation-tokenized                                                                                                                                                                                                                                                                                          | space-after-information                                                                                                                                                                                                                                                                                                                                                                                                       | fwd_align_dict                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         | bwd_align_dict                                                                                                                                                                                                                                                                                                                                                                                                                                                                         | alignments                                                                                                                                                                                                                                                                                                                                                                                   | new_translations                                                                                                                                                                                                                                                                                               | substitution_info                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   | substituted_pairs                                  | substituted_words                | errors   | source_indices                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |\n",
      "|---:|:------------------------------------------------------------------------------------------------------------------------------------------|:-----------------------------------------------|:---------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------:|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------|:---------------------------------|:---------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|\n",
      "|  2 | /home/tajak/Parlamint-translation/Source-data/ParlaMint-SI.conllu/ParlaMint-SI.conllu/2010/ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.ana.seg3.1 | Obveščen sem, da se današnje seje ne morejo udeležiti naslednja poslanka in poslanci: Eva Irgl, Zmago Jelinčič Plemeniti, Rudolf Petan, Roberto Battelli in tukaj imam napisano dr. Vinko Gorenak, ki pa je tukaj – od 16. ure dalje se Vinko Gorenak ne more udeležiti seje. | Obveščen sem , da se današnje seje ne morejo udeležiti naslednja poslanka in poslanci : Eva Irgl , Zmago Jelinčič Plemeniti , Rudolf Petan , Roberto Battelli in tukaj imam napisano dr. Vinko Gorenak , ki pa je tukaj – od 16. ure dalje se Vinko Gorenak ne more udeležiti seje . | {15: ['Eva', 'Eva'], 16: ['Irgl', 'Irgl'], 18: ['Zmago', 'Zmago'], 19: ['Jelinčič', 'Jelinčič'], 20: ['Plemeniti', 'Plemeniti'], 22: ['Rudolf', 'Rudolf'], 23: ['Petan', 'Petan'], 25: ['Roberto', 'Roberto'], 26: ['Battelli', 'Battelli'], 32: ['Vinko', 'Vinko'], 33: ['Gorenak', 'Gorenak'], 45: ['Vinko', 'Vinko'], 46: ['Gorenak', 'Gorenak']} |       45 | I am informed that today's meetings cannot be attended by the following Members and Members: Eva Irgl, Victory Jelinčič Trimeniti, Rudolf Petan, Roberto Battelli and here I have written Dr. Vinko Gorenak, who is here – from 16:00 onwards Vinko Gorenak cannot attend the meeting.                 | I am informed that today 's meetings cannot be attended by the following Members and Members : Eva Irgl , Victory Jelinčič Trimeniti , Rudolf Petan , Roberto Battelli and here I have written Dr. Vinko Gorenak , who is here – from 16:00 onwards Vinko Gorenak cannot attend the meeting .                  | ['Yes', 'Yes', 'Yes', 'Yes', 'No', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'No', 'Yes', 'Yes', 'No', 'Yes', 'Yes', 'Yes', 'No', 'Yes', 'Yes', 'No', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'No', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'No', 'Last']                                                        | {1: '2', 2: '1', 3: '1', 4: '4', 5: '6', 6: '6', 7: '7', 8: '9', 9: '10', 10: '10', 11: '10', 12: '11', 13: '11', 14: '12', 15: '13', 16: '14', 17: '15', 18: '16', 19: '17', 20: '18', 21: '19', 22: '20', 23: '21', 24: '22', 25: '23', 26: '24', 27: '25', 28: '26', 29: '27', 30: '28', 31: '29', 32: '30', 33: '30', 34: '31', 35: '32', 36: '33', 37: '34', 38: '35', 39: '36', 40: '38', 41: '39', 42: '40', 43: '41', 44: '42', 45: '44', 46: '46', 47: '47', 48: '49', 49: '50', 51: '51', 52: '52'}          | {3: '1', 1: '2', 2: '3', 4: '4', 6: '5', 5: '6', 7: '7', 8: '8', 9: '9', 10: '10', 13: '11', 14: '12', 15: '13', 16: '14', 17: '15', 18: '16', 19: '17', 20: '18', 21: '19', 22: '20', 23: '21', 24: '22', 25: '23', 26: '24', 27: '25', 28: '26', 29: '27', 30: '28', 31: '29', 33: '30', 34: '31', 35: '32', 36: '33', 37: '34', 38: '35', 39: '36', 40: '38', 41: '39', 42: '40', 43: '41', 44: '42, 43', 45: '44', 46: '46', 47: '47', 48: '48, 49', 49: '50', 51: '51', 52: '52'} | {0: 2, 1: 0, 2: 1, 3: 3, 4: 5, 5: 4, 6: 6, 7: 7, 8: 8, 9: 9, 10: 12, 11: 13, 12: 14, 13: 15, 14: 16, 15: 17, 16: 18, 17: 19, 18: 20, 19: 21, 20: 22, 21: 23, 22: 24, 23: 25, 24: 26, 25: 27, 26: 28, 27: 29, 28: 30, 29: 32, 30: 33, 31: 34, 32: 35, 33: 36, 34: 37, 35: 38, 37: 39, 38: 40, 39: 41, 40: 42, 41: 43, 42: 43, 43: 44, 45: 45, 46: 46, 47: 47, 48: 47, 49: 48, 50: 50, 51: 51} | I am informed that today 's meetings cannot be attended by the following Members and Members : Eva Irgl , Zmago Jelinčič Plemeniti , Rudolf Petan , Roberto Battelli and here I have written Dr. Vinko Gorenak , who is here – from 16:00 onwards Vinko Gorenak cannot attend the meeting .                    | [\"No substitution: ('Eva', 'Eva')\", \"No substitution: ('Irgl', 'Irgl')\", ('Victory', 'Zmago'), \"No substitution: ('Jelinčič', 'Jelinčič')\", ('Trimeniti', 'Plemeniti'), \"No substitution: ('Rudolf', 'Rudolf')\", \"No substitution: ('Petan', 'Petan')\", \"No substitution: ('Roberto', 'Roberto')\", \"No substitution: ('Battelli', 'Battelli')\", \"No substitution: ('Vinko', 'Vinko')\", \"No substitution: ('Gorenak', 'Gorenak')\", \"No substitution: ('Vinko', 'Vinko')\", \"No substitution: ('Gorenak', 'Gorenak')\"] | [('Victory', 'Zmago'), ('Trimeniti', 'Plemeniti')] | {21: 'Victory', 23: 'Trimeniti'} | No       | [['Obveščen', 1], ['sem', 2], [',', 3], ['da', 4], ['se', 5], ['današnje', 6], ['seje', 7], ['ne', 8], ['morejo', 9], ['udeležiti', 10], ['naslednja', 11], ['poslanka', 12], ['in', 13], ['poslanci', 14], [':', 15], ['Eva', 16], ['Irgl', 17], [',', 18], ['Zmago', 19], ['Jelinčič', 20], ['Plemeniti', 21], [',', 22], ['Rudolf', 23], ['Petan', 24], [',', 25], ['Roberto', 26], ['Battelli', 27], ['in', 28], ['tukaj', 29], ['imam', 30], ['napisano', 31], ['dr.', 32], ['Vinko', 33], ['Gorenak', 34], [',', 35], ['ki', 36], ['pa', 37], ['je', 38], ['tukaj', 39], ['–', 40], ['od', 41], ['16.', 42], ['ure', 43], ['dalje', 44], ['se', 45], ['Vinko', 46], ['Gorenak', 47], ['ne', 48], ['more', 49], ['udeležiti', 50], ['seje', 51], ['.', 52]] |\n",
      "|  8 | /home/tajak/Parlamint-translation/Source-data/ParlaMint-SI.conllu/ParlaMint-SI.conllu/2010/ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu | ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.ana.seg7.3 | Skupina poslank in poslancev s prvopodpisano Melito Župevc predlaga Državnemu zboru, da dnevni red seje razširi z obravnavo Ponovnega odločanja o Zakonu o spremembah in dopolnitvah Zakona o zemljiški knjigi, EPA 1152 /V. Predlog ste prejeli z dopisom z dne 2. 12. 2010. | Skupina poslank in poslancev s prvopodpisano Melito Župevc predlaga Državnemu zboru , da dnevni red seje razširi z obravnavo Ponovnega odločanja o Zakonu o spremembah in dopolnitvah Zakona o zemljiški knjigi , EPA 1152 / V. Predlog ste prejeli z dopisom z dne 2. 12. 2010 .    | {6: ['Melito', 'Melita'], 7: ['Župevc', 'Župevc']}                                                                                                                                                                                                                                                                                                   |       43 | The Group of Members and Members, signed first by Melito Župevc, proposes to the National Assembly that the agenda be extended by the consideration of the Redecision on the Law on Amendments and Supplements to the Law on Land Book, EPA 1152/V. You received the proposal by letter of 2. 12. 2010 | The Group of Members and Members , signed first by Melito Župevc , proposes to the National Assembly that the agenda be extended by the consideration of the Redecision on the Law on Amendments and Supplements to the Law on Land Book , EPA 1152 / V . You received the proposal by letter of 2 . 12 . 2010 | ['Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'No', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'No', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'No', 'Yes', 'Yes', 'No', 'No', 'No', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'No', 'Yes', 'No', 'Yes', 'Last'] | {2: '1', 4: '2', 5: '3', 6: '4', 7: '5', 8: '6', 9: '6', 10: '6', 11: '7', 12: '8', 13: '12', 14: '9', 15: '10', 17: '10', 18: '11', 19: '13', 20: '15', 21: '14', 22: '16', 23: '17', 24: '18', 26: '19', 29: '20', 30: '22', 32: '23', 33: '24', 34: '25', 35: '26', 36: '27', 37: '27', 39: '28', 40: '29', 41: '30', 42: '31', 43: '32', 44: '33', 45: '34', 46: '35', 47: '36', 48: '47', 49: '38', 50: '39', 51: '42', 52: '37', 53: '40', 54: '41', 55: '43', 56: '44', 57: '45', 58: '45', 59: '45', 60: '46'} | {2: '1', 4: '2', 5: '3', 6: '4', 7: '5', 8: '6', 11: '7', 12: '8', 14: '9', 17: '10', 18: '11', 13: '12', 19: '13', 21: '14, 15', 22: '16', 23: '17', 24: '18', 26: '19', 29: '20, 21', 30: '22', 32: '23', 33: '24', 34: '25', 35: '26', 36: '27', 39: '28', 40: '29', 41: '30', 42: '31', 43: '32', 44: '33', 45: '34', 46: '35', 47: '36', 52: '37', 49: '38', 50: '39', 51: '40', 54: '41', 53: '42', 55: '43', 56: '44', 58: '45', 60: '46', 59: '47'}                            | {0: 1, 1: 3, 2: 4, 3: 5, 4: 6, 5: 7, 6: 10, 7: 11, 8: 13, 9: 16, 10: 17, 11: 12, 12: 18, 13: 20, 14: 20, 15: 21, 16: 22, 17: 23, 18: 25, 19: 28, 20: 28, 21: 29, 22: 31, 23: 32, 24: 33, 25: 34, 26: 35, 27: 38, 28: 39, 29: 40, 30: 41, 31: 42, 32: 43, 33: 44, 34: 45, 35: 46, 36: 51, 37: 48, 38: 49, 39: 50, 40: 53, 41: 52, 42: 54, 43: 55, 44: 57, 45: 59, 46: 58}                     | The Group of Members and Members , signed first by Melita Župevc , proposes to the National Assembly that the agenda be extended by the consideration of the Redecision on the Law on Amendments and Supplements to the Law on Land Book , EPA 1152 / V . You received the proposal by letter of 2 . 12 . 2010 | [('Melito', 'Melita'), \"No substitution: ('Župevc', 'Župevc')\"]                                                                                                                                                                                                                                                                                                                                                                                                                                                     | [('Melito', 'Melita')]                             | {11: 'Melito'}                   | No       | [['Skupina', 1], ['poslank', 2], ['in', 3], ['poslancev', 4], ['s', 5], ['prvopodpisano', 6], ['Melito', 7], ['Župevc', 8], ['predlaga', 9], ['Državnemu', 10], ['zboru', 11], [',', 12], ['da', 13], ['dnevni', 14], ['red', 15], ['seje', 16], ['razširi', 17], ['z', 18], ['obravnavo', 19], ['Ponovnega', 20], ['odločanja', 21], ['o', 22], ['Zakonu', 23], ['o', 24], ['spremembah', 25], ['in', 26], ['dopolnitvah', 27], ['Zakona', 28], ['o', 29], ['zemljiški', 30], ['knjigi', 31], [',', 32], ['EPA', 33], ['1152', 34], ['/', 35], ['V.', 36], ['Predlog', 37], ['ste', 38], ['prejeli', 39], ['z', 40], ['dopisom', 41], ['z', 42], ['dne', 43], ['2.', 44], ['12.', 45], ['2010', 46], ['.', 47]]                                                 |\n"
     ]
    }
   ],
   "source": [
    "# Create a dictionary from the returned alignment files which will be added to each word in the final conllu\n",
    "\n",
    "def alignment_file_to_target_dict(file):\n",
    "\t\"\"\"\n",
    "\tThe output of the eflomal aligner is in the source to target direction. We want to get the alignments in the other direction\n",
    "\tand for each target word add to the conllu its aligned source word index (as it appears in conllu). In conllu, indices start\n",
    "\twith 1, not 0. So, we take the eflomal files, reverse the order and create dictionaries with target indexes as keys\n",
    "\tand source indexes as values. If there are more than one words aligned to the same target word, it looks like this: '1, 2'.\n",
    "\tWe use the conllu indexes which means that we add 1 to each index in the alingment pairs. \n",
    "\n",
    "\tArgs:\n",
    "\t\t- file: the path to the .fwd and .rev file that is produced by the eflomal tool\n",
    "\n",
    "\tThe result is a list of dictionaries, each dictionary corresponds to one sentence.\n",
    "\t\"\"\"\n",
    "\t# Create target alignments from the source alignment direction (by changing the direction in the file)\n",
    "\taligns_list_target = open(file, \"r\").readlines()\n",
    "\taligns_list_target = [i.replace(\"\\n\", \"\") for i in aligns_list_target]\n",
    "\taligns_list_target = [i.split(\" \") for i in aligns_list_target]\n",
    "\n",
    "\taligns_list_target_dict_list = []\n",
    "\n",
    "\t# Loop through the alignments for sentences\n",
    "\tfor i in aligns_list_target:\n",
    "\t\t# Create a dictionary for each sentence\n",
    "\t\tcurrent_sentence_align = {}\n",
    "\t\t# If alignment line is empty, keep the dictionary empty\n",
    "\t\tif len(i) == 1 and len(i[0]) == 0:\n",
    "\t\t\tcurrent_sentence_align = {}\n",
    "\t\telse:\n",
    "\t\t\t# For each alignment pair in the sentence:\n",
    "\t\t\tfor pair in i:\n",
    "\t\t\t\t# Split the pair: result is a list of lists with source index as the first element\n",
    "\t\t\t\t# and target index as the second element: [[0,0], [1,2], [1,3]]\n",
    "\t\t\t\tcurrent_pair = pair.split(\"-\")\n",
    "\n",
    "\t\t\t\t# Get the indices for target and source and add 1 to them (to get the conllu indices)\n",
    "\t\t\t\tcurrent_t_index = int(current_pair[1]) + 1\n",
    "\t\t\t\tcurrent_s_index = int(current_pair[0]) + 1\n",
    "\n",
    "\t\t\t\t# Check whether the target index is already aligned to anything (a case of 1-to-many alignment),\n",
    "\t\t\t\t# if not, save it as a key and save the source index as value.\n",
    "\t\t\t\tif current_sentence_align.get(current_t_index, None) == None:\n",
    "\t\t\t\t\tcurrent_sentence_align[current_t_index] = str(current_s_index)\n",
    "\t\t\t\t# If the index was already aligned to a previous source word, add the additional source word alignment as a string\n",
    "\t\t\t\t# (result: {0: \"1, 2\"))\n",
    "\t\t\t\telse:\n",
    "\t\t\t\t\tcurrent_sentence_align[current_t_index] += str(\", \")\n",
    "\t\t\t\t\tcurrent_sentence_align[current_t_index] += str(current_s_index)\n",
    "\n",
    "\t\taligns_list_target_dict_list.append(current_sentence_align)\n",
    "\n",
    "\treturn aligns_list_target_dict_list\n",
    "\n",
    "def correct_proper_nouns(translated_tokenized_dataframe_path, final_dataframe, lang_code):\n",
    "\t\"\"\"\n",
    "\tThis function takes the translated text and the source text, aligns words with eflomal and corrects proper nouns.\n",
    "\tIt takes the dataframe that was created in the function extract_text() and to which the translation was added\n",
    "\tin the function translate().\n",
    "\n",
    "\tTo use eflomal, you need to install it first:\n",
    "\t!git clone https://github.com/robertostling/eflomal\n",
    "\t%cd eflomal\n",
    "\t!make\n",
    "\t!sudo make install\n",
    "\t!python3 setup.py install\n",
    "\n",
    "\tIn case you don't have sudo permission, you can skip !sudo make install. I did, and I also used a virtual environment (venv), and managed to install eflomal.\n",
    "\n",
    "\t\"\"\"\n",
    "\timport pandas as pd\n",
    "\timport re\n",
    "\timport ast\n",
    "\tfrom IPython.display import display\n",
    "\timport time\n",
    "\timport subprocess\n",
    "\timport os\n",
    "\n",
    "\t# Open the file, created in the previous step\n",
    "\tdf = pd.read_csv(\"{}\".format(translated_tokenized_dataframe_path), sep=\"\\t\", index_col=0)\n",
    "\n",
    "\t# Move into the eflomal folder\n",
    "\tos.chdir(\"/home/tajak/Parlamint-translation/eflomal\")\n",
    "\n",
    "\t# Then we need to create files for all texts and all translations\n",
    "\tsource_sentences = open(\"source_sentences_{}_missing_file.txt\".format(lang_code), \"w\")\n",
    "\tEnglish_sentences = open(\"English_sentences_{}_missing_file.txt\".format(lang_code), \"w\")\n",
    "\n",
    "\tfor i in df[\"tokenized_text\"].to_list():\n",
    "\t\tsource_sentences.write(i)\n",
    "\t\tsource_sentences.write(\"\\n\")\n",
    "\n",
    "\tfor i in df[\"translation-tokenized\"].to_list():\n",
    "\t\tEnglish_sentences.write(i)\n",
    "\t\tEnglish_sentences.write(\"\\n\")\n",
    "\n",
    "\tsource_sentences.close()\n",
    "\tEnglish_sentences.close()\n",
    "\n",
    "\tprint(\"\\n\\n\")\n",
    "\tprint(\"Alignment started.\")\n",
    "\tstart_time = time.time()\n",
    "\n",
    "\t# Align sentences with eflomal and get out a file with alignments\n",
    "\t!python3 align.py -s \"source_sentences_SI_missing_file.txt\" -t \"English_sentences_SI_missing_file.txt\" --priors si-en.priors --model 3 -f source-en-SI-add-file.fwd -r source-en-SI-add-file.rev\n",
    "\n",
    "\t# Create a list of dictionaries of alignments from the returned files which will be added to the final conllu for each word\n",
    "\tforward_alignment_dict_list = alignment_file_to_target_dict(\"source-en-{}-add-file.fwd\".format(lang_code))\n",
    "\tbackward_alignment_dict_list = alignment_file_to_target_dict(\"source-en-{}-add-file.rev\".format(lang_code))\n",
    "\n",
    "\t# Add to the df\n",
    "\tdf[\"fwd_align_dict\"] = forward_alignment_dict_list\n",
    "\tdf[\"bwd_align_dict\"] = backward_alignment_dict_list\n",
    "\n",
    "\t# Create forward target alignments from the source alignment direction (by changing the direction in the rev file)\n",
    "\taligns_list = open(\"source-en-{}-add-file.rev\".format(lang_code), \"r\").readlines()\n",
    "\taligns_list = [i.replace(\"\\n\", \"\") for i in aligns_list]\n",
    "\n",
    "\t# Continue with processing the list to create the final alignments format which I'll use to correct proper names\n",
    "\taligns_list = [i.split(\" \") for i in aligns_list]\n",
    "\n",
    "\tfor i in aligns_list:\n",
    "\t\t# If alignment line is empty, keep the dictionary empty\n",
    "\t\tif len(i) == 1 and len(i[0]) == 0:\n",
    "\t\t\taligns_list[aligns_list.index(i)] = []\n",
    "\t\telse:\n",
    "\t\t\tfor pair in i:\n",
    "\t\t\t\tcurrent_pair = pair.split(\"-\")\n",
    "\t\t\t\ti[i.index(pair)] = {int(current_pair[0]): int(current_pair[1])}\n",
    "\t\n",
    "\tfinal_aligns = []\n",
    "\n",
    "\t# Create a dictionary out of the rev alignments\n",
    "\tfor i in aligns_list:\n",
    "\t\tcurrent_line = {}\n",
    "\n",
    "\t\ttry:\n",
    "\t\t\tfor element in i:\n",
    "\t\t\t\ta = list(element.items())[0][0]\n",
    "\t\t\t\tb = list(element.items())[0][1]\n",
    "\t\t\t\tcurrent_line[a] = b\n",
    "\t\t\n",
    "\t\t\t# Check whether the number of pairs in the list is the same as number of items\n",
    "\t\t\tif len(i) != len(list(current_line.items())):\n",
    "\t\t\t\tprint(\"Not okay:\")\n",
    "\t\t\t\tprint(i)\n",
    "\t\t\t\tprint(current_line)\n",
    "\n",
    "\t\t\tfinal_aligns.append(current_line)\n",
    "\t\t\n",
    "\t\texcept:\n",
    "\t\t\tprint(\"error\")\n",
    "\t\t\tprint(aligns_list.index(i))\n",
    "\t\t\tprint(i)\n",
    "\t\t\tfinal_aligns.append(\"Error\")\n",
    "\t\t\n",
    "\tprint(\"\\nNumber of aligned sentences: {}\\n\\n\".format(len(final_aligns)))\n",
    "\n",
    "\t# Add a to the df\n",
    "\tdf[\"alignments\"] = final_aligns\n",
    "\n",
    "\t# When we open the dataframe file, the dictionaries with proper names changed into strings - Change strings in the column proper_nouns into dictionaries\n",
    "\n",
    "\tdf[\"proper_nouns\"] = df.proper_nouns.astype(\"str\")\n",
    "\tdf[\"proper_nouns\"] = df.proper_nouns.apply(lambda x: ast.literal_eval(x))\n",
    "\n",
    "\t# Change nan values in the proper_nouns columns\n",
    "\tdf = df.fillna(0)\n",
    "\n",
    "\t# Substitute words in the translation based on alignments\n",
    "\tintermediate_list = list(zip(df[\"translation-tokenized\"], df[\"proper_nouns\"], df[\"alignments\"]))\n",
    "\n",
    "\tnew_translations = []\n",
    "\tsubstituted_all_info = []\n",
    "\tsubstituted_only = []\n",
    "\tsubstituted_words = []\n",
    "\n",
    "\t# Add information whether an error occurred\n",
    "\terror_list = []\n",
    "\n",
    "\tfor i in intermediate_list:\n",
    "\t\tcurrent_substituted_list = []\n",
    "\t\tcurrent_substituted_only = []\n",
    "\t\tcurrent_substituted_words = {}\n",
    "\t\tcurrent_error = \"No\"\n",
    "\n",
    "\t\t# If no proper names were detected, do not change the translation\n",
    "\t\tif i[1] == 0:\n",
    "\t\t\tnew_translations.append(i[0])\n",
    "\t\t\n",
    "\t\telse:\n",
    "\t\t\tcurrent_translation = i[0]\n",
    "\n",
    "\t\t\t# Substitute the word with the source lemma based on the index - loop through the proper nouns to be changed\n",
    "\t\t\tfor word_index in list(i[1].keys()):\n",
    "\t\t\t\ttry:\n",
    "\t\t\t\t\t# split the translation into list of words\n",
    "\t\t\t\t\tword_list = current_translation.split()\n",
    "\n",
    "\t\t\t\t\t# Get index of the substituted word\n",
    "\t\t\t\t\tsubstituted_word_index = i[2][word_index]\n",
    "\n",
    "\t\t\t\t\t# Get the lemma to substitute the word with\n",
    "\t\t\t\t\tcorrect_lemma = i[1][word_index][1]\n",
    "\n",
    "\t\t\t\t\t# If the substitute word and lemma are not the same, get substituted word and its match\n",
    "\t\t\t\t\tif word_list[substituted_word_index] != correct_lemma:\n",
    "\t\t\t\t\t\tcurrent_substituted_list.append((word_list[substituted_word_index], correct_lemma))\n",
    "\t\t\t\t\t\tcurrent_substituted_only.append((word_list[substituted_word_index], correct_lemma))\n",
    "\n",
    "\t\t\t\t\t\t# Save information on which word was substituted with its conllu index (index + 1) as the key\n",
    "\t\t\t\t\t\tcurrent_substituted_words[int(substituted_word_index+1)] = word_list[substituted_word_index]\n",
    "\n",
    "\t\t\t\t\t\t# Substitute the word in the word list\n",
    "\t\t\t\t\t\tword_list[substituted_word_index] = correct_lemma\n",
    "\t\t\t\t\t\n",
    "\t\t\t\t\telse:\n",
    "\t\t\t\t\t\t# Add information that substitution was not performed\n",
    "\t\t\t\t\t\tcurrent_substituted_list.append(f\"No substitution: {word_list[substituted_word_index], correct_lemma}\")\n",
    "\t\t\t\t\t\n",
    "\t\t\t\t\t# Change the translation by merging the words back into a string\n",
    "\t\t\t\t\tcurrent_translation = \" \".join(word_list)\n",
    "\n",
    "\t\t\t\texcept:\n",
    "\t\t\t\t\tprint(f\"Issue: index {word_index}: {i[1][word_index]}\")\n",
    "\t\t\t\t\tcurrent_error = f\"Issue: index {word_index}: {i[1][word_index]}\"\n",
    "\n",
    "\t\t\t# After the loop through proper nouns, save the new translation\n",
    "\t\t\tnew_translations.append(current_translation)\n",
    "\t\t\n",
    "\t\t# Add information on what was substituted\n",
    "\t\tif len(substituted_all_info) != 0:\n",
    "\t\t\tsubstituted_all_info.append(current_substituted_list)\n",
    "\t\telse:\n",
    "\t\t\tsubstituted_all_info.append(0)\n",
    "\n",
    "\t\tif len(current_substituted_only) != 0:\n",
    "\t\t\tsubstituted_only.append(current_substituted_only)\n",
    "\t\telse:\n",
    "\t\t\tsubstituted_only.append(0)\n",
    "\n",
    "\t\terror_list.append(current_error)\n",
    "\n",
    "\t\tsubstituted_words.append(current_substituted_words)\n",
    "\n",
    "\n",
    "\t# Add to the df\n",
    "\tdf[\"new_translations\"] = new_translations\n",
    "\tdf[\"substitution_info\"] = substituted_all_info\n",
    "\tdf[\"substituted_pairs\"] = substituted_only\n",
    "\tdf[\"substituted_words\"] = substituted_words\n",
    "\tdf[\"errors\"] = error_list\n",
    "\n",
    "\tend_time = round((time.time() - start_time)/60,2)\n",
    "\n",
    "\tprint(\"Alignment completed. It took {} minutes.\".format(end_time))\n",
    "\n",
    "\t# Change the working directory once again\n",
    "\tos.chdir(\"..\")\n",
    "\n",
    "\t# Add the word list with indices to the df\n",
    "\ttokenized_text_list = df.tokenized_text.to_list()\n",
    "\ttokenized_text_list = [i.split(\" \") for i in tokenized_text_list]\n",
    "\ttokenized_text_dict_list = []\n",
    "\n",
    "\tfor sentence in tokenized_text_list:\n",
    "\t\tsentence_list = []\n",
    "\t\tcounter = 1\n",
    "\t\tfor word in sentence:\n",
    "\t\t\tsentence_list.append([word, counter])\n",
    "\t\t\tcounter += 1\n",
    "\t\ttokenized_text_dict_list.append(sentence_list)\n",
    "\n",
    "\tdf[\"source_indices\"] = tokenized_text_dict_list\n",
    "\n",
    "\t# Save the df\n",
    "\tdf.to_csv(\"{}\".format(final_dataframe), sep=\"\\t\")\n",
    "\n",
    "\t# Display most common substitutions\n",
    "\tdf_substituted = df[df[\"proper_nouns\"] != \"0\"]\n",
    "\tprint(df_substituted.substituted_pairs.value_counts()[:20].to_markdown())\n",
    "\tprint(\"\\n\\n\")\n",
    "\n",
    "\treturn df\n",
    "\n",
    "\n",
    "\n",
    "df = correct_proper_nouns(translated_tokenized_dataframe_path, final_dataframe, lang_code)\n",
    "\n",
    "# See if there were any errors in word substitution\n",
    "print(\"Number of errors:\")\n",
    "print(df[df[\"errors\"]!=\"No\"].shape)\n",
    "print(\"\\n\\n\")\n",
    "\n",
    "# See example of sentences with substituted words\n",
    "print(\"Example of sentences with substituted words.\")\n",
    "print(df[df[\"substituted_pairs\"]!= 0][:2].to_markdown())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "create_conllu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-10 12:27:45 WARNING: Can not find mwt: default from official model list. Ignoring it.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing started.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-10 12:27:45 INFO: Loading these models for language: en (English):\n",
      "========================\n",
      "| Processor | Package  |\n",
      "------------------------\n",
      "| tokenize  | combined |\n",
      "| pos       | combined |\n",
      "| lemma     | combined |\n",
      "| ner       | conll03  |\n",
      "========================\n",
      "\n",
      "2023-03-10 12:27:45 INFO: Use device: gpu\n",
      "2023-03-10 12:27:45 INFO: Loading: tokenize\n",
      "2023-03-10 12:27:45 INFO: Loading: pos\n",
      "2023-03-10 12:27:46 INFO: Loading: lemma\n",
      "2023-03-10 12:27:46 INFO: Loading: ner\n",
      "2023-03-10 12:27:46 INFO: Done loading processors!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu processed and saved.\n",
      "Error based on current_space after in sentence TokenList<In, understanding, all, these, circumstances, ,, on, the, basis, of, the, information, I, have, been, able, to, obtain, ,, to, think, through, ,, and, to, correct, ,, even, in, the, debates, to, complement, both, the, professional, services, and, the, professional, public, ,, both, the, lazi, public, in, the, Slovenian, space, ,, I, will, somehow, assess, that, the, measure, in, this, direction, ,, as, it, was, recorded, before, the, adoption, of, this, amendment, ,, is, a, measure, that, balances, the, principle, of, fairness, among, generations, so, that, ,, in, the, other, solutions, that, this, law, brings, ,, we, can, balance, these, burdens, among, all, generations, ,, provide, the, motive, that, the, younger, ones, remain, in, the, solidarity, system, ,, that, we, are, willing, to, give, a, certain, value, to, contributions, to, pension, commodities, ,, so, that, ,, also, on, the, basis, of, this, same, solidarity, ,, we, will, be, assured, to, survive, a, decent, age, in, the, future, and, ,, of, course, ,, the, motive, for, young, people, entering, the, labour, relationship, ,, that, it, is, reasonable, to, maintain, this, system, of, intergenerational, solidarity, and, also, solidarity, among, the, rich, and, poor, in, the, future, ,, of, which, is, adopted, by, the, early, speech4, ., metadata={sent_id: \"ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.ana.seg542.1\", source: \"V razumevanju vseh teh okoliščin, na podlagi podatkov, ki sem jih imel možnost dobiti, premisliti, tudi korigirati, tudi v razpravah dopolnjevati tako v okviru strokovnih služb, tako strokovne javnosti, tako laične javnosti v slovenskem prostoru, nekako ocenim, da je ukrep v tej smeri, kot je bil zapisan pred sprejetjem tega amandmaja, tisti ukrep, ki uravnoteži načelo pravičnosti med generacijam, da bomo lahko ob drugih rešitvah, ki jih ta zakon prinaša, uravnoteženo porazdelili ta bremena med vse generacije, zagotovili motiv, da mlajši ostajamo v sistemu solidarnosti, da smo pripravljeni dajati določeno vrednost prispevkov v pokojninsko blagajno, zato da bo tudi nam na podlagi te iste solidarnosti v prihodnje zagotovljeno preživljati dostojno starost in seveda motiv mladim, ki vstopajo v delovno razmerje, da je smiselno ohranjati ta sistem medgeneracijske solidarnosti in tudi solidarnosti med bogatimi in revnimi, o kateri se manj govori, s sprejeto zgodnjo omejitvijo pokojnin v razmerju 1:4.\"}>, sentence index: 1738, word ., word index 203.\n",
      "Final file /home/tajak/Parlamint-translation/Final-data/ParlaMint-SI.conllu/ParlaMint-SI.conllu/2010/ParlaMint-SI_2010-12-02-SDZ5-Izredna-34.conllu is saved.\n",
      "Current running time: 0.38\n",
      "Processing completed. It took 0.38 minutes.\n"
     ]
    }
   ],
   "source": [
    "def create_conllu(file, lang_code, main_path, final_dataframe, nlp):\n",
    "\t\"\"\"\n",
    "\tThe function takes the dataframe (df), created in previous steps and takes only the instances from the df that belong\n",
    "\tto the file that is in the argument. It linguistically processes the translated sentences from the file and saves the file.\n",
    "\tThen we add additional information (metadata and NER annotations) to it with the conllu parser and save the final conllu file.\n",
    "\n",
    "\tArgs:\n",
    "\t\t- file (str): file name from the files list (see above)\n",
    "\t\t- lang_code (str): the lang code that is used in the names of the files, it should be the same as for extract_text()\n",
    "\t\"\"\"\n",
    "\n",
    "\t# Process all sentences in the dataframe and save them to a conllu file\n",
    "\tfrom stanza.utils.conll import CoNLL\n",
    "\timport stanza\n",
    "\tfrom conllu import parse\n",
    "\timport ast\n",
    "\timport regex as re\n",
    "\timport os\n",
    "\timport pandas as pd\n",
    "\n",
    "\t# Use the dataframe, created in previous steps\n",
    "\tdf = pd.read_csv(\"{}\".format(final_dataframe), sep=\"\\t\", index_col = 0)\n",
    "\n",
    "\t# Filter out only instances from the file in question\n",
    "\tdf = df[df[\"file\"] == file]\n",
    "\n",
    "\t# Add information on the target path\n",
    "\tdf[\"target_path\"] = df.file_path.str.replace(\"Source-data\", \"Final-data\")\n",
    "\n",
    "\t# Get target path\n",
    "\ttarget_path = list(df.target_path.unique())[0]\n",
    "\n",
    "\t# When we open the dataframe file, the lists and dictionaries turn into strings - change them back\n",
    "\tfor column in [\"space-after-information\", 'fwd_align_dict', 'bwd_align_dict', 'substituted_words', \"source_indices\"]:\n",
    "\t\tdf[column] = df[column].astype(\"str\")\n",
    "\t\tdf[column] = df[column].apply(lambda x: ast.literal_eval(x))\n",
    "\n",
    "\t# Create lists of information that we need to add to the conllu file\n",
    "\tids_list = df.sentence_id.to_list()\n",
    "\tsource_text = df.text.to_list()\n",
    "\t# initial_translation = df.translation.to_list()\n",
    "\tspace_after_list = df[\"space-after-information\"].to_list()\n",
    "\tfwd_align_list = df['fwd_align_dict'].to_list()\n",
    "\tbwd_align_list = df['bwd_align_dict'].to_list()\n",
    "\tsubstituted_words_list = df['substituted_words'].to_list()\n",
    "\t# tokenized_text_list = df[\"source_indices\"].to_list()\n",
    "\tsentence_list = df.new_translations.to_list()\n",
    "\n",
    "\t# To feed the entire list into the pipeline, we need to create lists of tokens, split by space\n",
    "\tsentence_list = [x.split(\" \") for x in sentence_list]\n",
    "\t\n",
    "\t# Linguistically process the list\n",
    "\tdoc = nlp(sentence_list)\n",
    "\n",
    "\t# Save the conllu file\n",
    "\tCoNLL.write_doc2conll(doc, \"{}/results/{}/temp/{}\".format(main_path, lang_code, file))\n",
    "\n",
    "\tprint(\"{} processed and saved.\".format(file))\n",
    "\n",
    "\t# Open the CONLL-u file with the CONLL-u parser\n",
    "\n",
    "\tdata = open(\"{}/results/{}/temp/{}\".format(main_path, lang_code, file), \"r\").read()\n",
    "\n",
    "\tsentences = parse(data)\n",
    "\n",
    "\t# Adding additional information to the conllu\n",
    "\tfor sentence in sentences:\n",
    "\t\t# Get the sentence index\n",
    "\t\tsentence_index = sentences.index(sentence)\n",
    "\n",
    "\t\t# Add metadata\n",
    "\t\tsentence.metadata[\"sent_id\"] = ids_list[sentence_index]\n",
    "\t\tsentence.metadata[\"source\"] = source_text[sentence_index]\n",
    "\t\t# sentence.metadata[\"source_indices\"] = tokenized_text_list[sentence_index]\n",
    "\t\t# sentence.metadata[\"initial_translation\"] = initial_translation[sentence_index]\n",
    "\n",
    "\t\t# Delete the current metadata for text\n",
    "\t\tdel sentence.metadata[\"text\"]\n",
    "\n",
    "\t\tnew_translation_text = \"\"\n",
    "\n",
    "\t\t# Iterate through tokens\n",
    "\t\tfor word in sentence:\n",
    "\t\t\tword_index = sentence.index(word)\n",
    "\t\t\tword_conllu_index = word[\"id\"]\n",
    "\n",
    "\t\t\t# Check whether the word conllu index (word id) is in the substituted_words_list (it is if it was substituted)\n",
    "\t\t\t# If it is, add information on the original translated word\n",
    "\t\t\tif substituted_words_list[sentence_index].get(word_conllu_index, None) != None:\n",
    "\t\t\t\tword[\"misc\"][\"Translated\"] = substituted_words_list[sentence_index][word_conllu_index]\n",
    "\t\t\t\n",
    "\t\t\t# Do the same for the forward and backward alignment\n",
    "\t\t\tif fwd_align_list[sentence_index].get(word_conllu_index, None) != None:\n",
    "\t\t\t\tword[\"misc\"][\"ForwardAlignment\"] = fwd_align_list[sentence_index][word_conllu_index]\n",
    "\n",
    "\t\t\tif bwd_align_list[sentence_index].get(word_conllu_index, None) != None:\n",
    "\t\t\t\tword[\"misc\"][\"BackwardAlignment\"] = bwd_align_list[sentence_index][word_conllu_index]\n",
    "\n",
    "\t\t\t# Remove information on start_char and end_char from the annotation\n",
    "\t\t\tdel word[\"misc\"][\"start_char\"]\n",
    "\t\t\tdel word[\"misc\"][\"end_char\"]\n",
    "\t\t\t\n",
    "\t\t\t# Change the NER tags so that they are the same as in the source\n",
    "\t\t\tcurrent_ner = word[\"misc\"][\"ner\"]\n",
    "\t\t\tdel word[\"misc\"][\"ner\"]\n",
    "\t\t\t\n",
    "\t\t\t# Substitute parts of the tags so that they are the same as in source\n",
    "\t\t\tcurrent_ner = re.sub(\"S-\", \"B-\", current_ner)\n",
    "\t\t\tcurrent_ner = re.sub(\"E-\", \"I-\", current_ner)\n",
    "\n",
    "\t\t\tword[\"misc\"][\"NER\"] = current_ner\n",
    "\n",
    "\t\t\ttry:\n",
    "\t\t\t\t# Get information about the space after based on the index\n",
    "\t\t\t\tcurrent_space_after = space_after_list[sentence_index][word_index]\n",
    "\t\t\texcept:\n",
    "\t\t\t\tprint(\"Error based on current_space after in sentence {}, sentence index: {}, word {}, word index {}.\".format(sentence, sentence_index, word, word_index))\n",
    "\t\t\t\tcurrent_space_after = \"Yes\"\n",
    "\n",
    "\t\t# Create new text from translation, correcting the spaces around words\n",
    "\t\t# based on the SpaceAfter information\n",
    "\t\t\tif current_space_after == \"No\":\n",
    "\t\t\t\tword[\"misc\"][\"SpaceAfter\"] = \"No\"\n",
    "\t\t\t\tnew_translation_text += word[\"form\"]\n",
    "\t\t\telif current_space_after == \"Last\":\n",
    "\t\t\t\tnew_translation_text += word[\"form\"]\n",
    "\t\t\telse:\n",
    "\t\t\t\tnew_translation_text += word[\"form\"]\n",
    "\t\t\t\tnew_translation_text += \" \"\n",
    "\t\t\n",
    "\t\tsentence.metadata[\"text\"] = new_translation_text\n",
    "\t\n",
    "\t# Create a new conllu file with the updated information\n",
    "\tos.makedirs(os.path.dirname(target_path), exist_ok=True)\n",
    "\tfinal_file = open(\"{}\".format(target_path), \"w\")\n",
    "\n",
    "\tfor sentence in sentences:\n",
    "\t\tfinal_file.write(sentence.serialize())\n",
    "\t\n",
    "\tfinal_file.close()\n",
    "\n",
    "\tprint(\"Final file {} is saved.\".format(target_path))\n",
    "\n",
    "def produce_final_conllu(lang_code, final_dataframe):\n",
    "\timport pandas as pd\n",
    "\timport stanza\n",
    "\timport time\n",
    "\tfrom stanza.pipeline.core import DownloadMethod\n",
    "\t\n",
    "\tdf = pd.read_csv(\"{}\".format(final_dataframe), sep=\"\\t\", index_col=0)\n",
    "\n",
    "\t# Create a list of files\n",
    "\tfiles = list(df.file.unique())\n",
    "\t\n",
    "\tstart_time = time.time()\n",
    "\n",
    "\tprint(\"Processing started.\")\n",
    "\n",
    "\t# Define the pipeline, instruct it to use a specific package: \tCoNLL03\n",
    "\tnlp = stanza.Pipeline(lang='en', processors=\"tokenize,mwt,pos,lemma,ner\", package={\"ner\": [\"conll03\"]}, tokenize_pretokenized=True, download_method=DownloadMethod.REUSE_RESOURCES, use_gpu=True)\n",
    "\n",
    "\tfor file in files:\n",
    "\t\tcreate_conllu(file, lang_code, main_path, final_dataframe, nlp)\n",
    "\t\tcurrent_end_time = round((time.time() - start_time)/60,2)\n",
    "\t\tprint(\"Current running time: {}\".format(current_end_time))\n",
    "\t\n",
    "\tend_time = round((time.time() - start_time)/60,2)\n",
    "\n",
    "\tprint(\"Processing completed. It took {} minutes.\".format(end_time))\n",
    "\n",
    "\n",
    "produce_final_conllu(lang_code, final_dataframe)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "parlamint_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "7e8622831a220209edc4d2d4e58bc2159575ea2f8d9419209393154757ff5f93"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}