parse_tree_approach.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "# import torch\n",
    "# from sklearn.metrics.pairwise import cosine_similarity\n",
    "# from tqdm import tqdm\n",
    "# from spacy.lang.fa import Persian\n",
    "# from spacy.lang.he import Hebrew\n",
    "# from spacy.lang.el import Greek\n",
    "import os\n",
    "# from POS.POSTagger import POSTagger\n",
    "\n",
    "# def ActivateModel(modelname=\"bert-base-multilingual-cased\"):\n",
    "#     from transformers import AutoConfig, AutoTokenizer, AutoModel\n",
    "\n",
    "#     # Activate model and tokenizer\n",
    "\n",
    "#     # \"HooshvareLab/bert-fa-base-uncased\"\n",
    "\n",
    "#     config = AutoConfig.from_pretrained(modelname)\n",
    "#     tokenizer = AutoTokenizer.from_pretrained(\"bert-base-multilingual-cased\")\n",
    "#     model = AutoModel.from_pretrained(modelname)\n",
    "#     return config, tokenizer, model\n",
    "\n",
    "def LoadBible(parquet_filename):\n",
    "    # Read pre-processed farsi bible as pandas dataframe\n",
    "    return pd.read_parquet(f\"transformations/{parquet_filename}.parquet\")\n",
    "\n",
    "def GenerateBibleWords(bible_df, target):     \n",
    "    for bk in sorted(set(bible_df.book)):\n",
    "        for ic in bible_df.loc[bible_df.book == bk].idx_chapter:\n",
    "            for iv in bible_df.loc[(bible_df.book == bk) & (bible_df.idx_chapter == ic)].idx_verse:\n",
    "                # Prepare subset of verse(s) for feeding to model\n",
    "                bible_words = (\n",
    "                    bible_df\n",
    "                    .loc[\n",
    "                        (bible_df.book == bk) &\n",
    "                        (bible_df.idx_chapter == ic) &\n",
    "                        (bible_df.idx_verse == iv)\n",
    "                    ]\n",
    "                )\n",
    "                if target == \"idx_word\":\n",
    "                    yield bible_words.idx_word.to_list(), (bk, ic, iv)\n",
    "                else:\n",
    "                    bible_words = (\n",
    "                        bible_words\n",
    "                        .groupby([\"book\", \"idx_chapter\", \"idx_verse\", \"idx_word\"])[target]\n",
    "                        .agg(\" \".join)\n",
    "                        .to_list()\n",
    "                    )\n",
    "                    split_bible_words = [a for sub in [s.split(\"/\") for s in bible_words] for a in sub]\n",
    "                    yield split_bible_words, (bk, ic, iv)\n",
    "\n",
    "# def GetWordEmbeddings(words, model, tokenizer):\n",
    "\n",
    "#     def mean_pooling(model_output, attention_mask):\n",
    "#         token_embeddings = model_output[0] #First element of model_output contains all token embeddings\n",
    "#         input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()\n",
    "#         sum_embeddings = torch.sum(token_embeddings * input_mask_expanded, 1)\n",
    "#         sum_mask = torch.clamp(input_mask_expanded.sum(1), min=1e-9)\n",
    "#         return sum_embeddings / sum_mask\n",
    "        \n",
    "#     # Tokenize sentences\n",
    "#     encoded_input = tokenizer(words, padding=True, truncation=False, return_tensors='pt')\n",
    "\n",
    "#     # Compute token embeddings\n",
    "#     with torch.no_grad():\n",
    "#         model_output = model(**encoded_input)\n",
    "#     embeddings = mean_pooling(model_output, encoded_input['attention_mask'])\n",
    "\n",
    "#     # words_embeddings_df = pd.DataFrame({\"word\": words, \"embeddings\": embeddings.tolist()})\n",
    "#     return [np.array(e).reshape(1,-1) for e in embeddings.tolist()]\n",
    "\n",
    "# # How to only process non stopwords and punctuation?\n",
    "# def max_cos_sim(row, model, tokenizer):\n",
    "#     ser = row.copy()\n",
    "\n",
    "#     ser[\"embeddings_nmv\"] = GetWordEmbeddings(ser.word, model, tokenizer)\n",
    "#     ser[\"embeddings_original\"] = GetWordEmbeddings(ser.eng_word, model, tokenizer)\n",
    "#     # if len(row[\"word\"]) > 1 or row[\"word\"] in ['آ', 'و', '\\u200c', '\\u200f']:\n",
    "#     similarity = pd.DataFrame(\n",
    "#         [\n",
    "#             (\n",
    "#                 ixo, \n",
    "#                 ixn,\n",
    "#                 wn,\n",
    "#                 ptn,\n",
    "#                 cosine_similarity(eo, en)\n",
    "#             ) \n",
    "#             for ixo, eo \n",
    "#             in zip(\n",
    "#                 ser.idx_word_original, \n",
    "#                 ser.embeddings_original\n",
    "#             )\n",
    "#             for ixn, wn, ptn, en \n",
    "#             in zip(\n",
    "#                 ser.idx_word_nmv,\n",
    "#                 ser.word,\n",
    "#                 ser.pos_tag,\n",
    "#                 ser.embeddings_nmv\n",
    "#             )\n",
    "#         ]\n",
    "#     )\n",
    "#     similarity.columns = [\"idx_word_original\", \"idx_wordmatch_nmv\", \"wordmatch_nmv\", \"pos_tag_nmv\", \"cosine_similarity\"]\n",
    "#     similarity[\"helper\"] = 0\n",
    "#     out = (\n",
    "#         similarity\n",
    "#         .sort_values(\"cosine_similarity\")\n",
    "#         .drop_duplicates(\n",
    "#             subset=\"idx_word_original\",\n",
    "#             keep=\"last\"\n",
    "#         )\n",
    "#         .groupby(\"helper\")\n",
    "#         [[\"idx_wordmatch_nmv\", \"wordmatch_nmv\", \"pos_tag_nmv\", \"cosine_similarity\"]]\n",
    "#         .agg(list)\n",
    "#     )\n",
    "#     # print(out.squeeze())\n",
    "#     return out.squeeze()\n",
    "\n",
    "# def GenerateMatches(\n",
    "#     idx_gen,\n",
    "#     nmv_gen,\n",
    "#     orig_gen,\n",
    "#     strongs_gen\n",
    "# ):\n",
    "# # figure out how to iterate through in parallel and avoid restarting generator each time. \n",
    "#     for idx_verse, nmv_verse, orig_verse, strongs in zip(\n",
    "#         idx_gen,\n",
    "#         nmv_gen,\n",
    "#         orig_gen,\n",
    "#         strongs_gen\n",
    "#     ):\n",
    "#         nmv_embeddings = GetWordEmbeddings(nmv_verse[0], model, tokenizer)\n",
    "#         nmv_embeddings[\"idx_word\"] = idx_verse[0]\n",
    "#         orig_embeddings = GetWordEmbeddings(orig_verse[0], model, tokenizer)\n",
    "#         orig_embeddings[\"strongs\"] = strongs[0]\n",
    "#         orig_embeddings[[\"idx_word\", \"similarity\"]] = orig_embeddings.apply(max_cos_sim, axis=1, args=[nmv_embeddings])\n",
    "#         nmv_embeddings = (\n",
    "#             nmv_embeddings\n",
    "#             .set_index(\"idx_word\")\n",
    "#             .join(\n",
    "#                 orig_embeddings\n",
    "#                 .drop(columns=\"embeddings\")\n",
    "#                 .set_index(\"idx_word\"),\n",
    "#                 how=\"left\",\n",
    "#                 rsuffix=\"_orig\"\n",
    "#             )\n",
    "#         )\n",
    "#         nmv_embeddings = (\n",
    "#             nmv_embeddings\n",
    "#             .reset_index()\n",
    "#             .sort_values(by=\"similarity\")\n",
    "#             .drop_duplicates(subset=\"idx_word\", keep=\"last\")\n",
    "#             .sort_values(by=\"idx_word\")\n",
    "#         )\n",
    "#         nmv_embeddings[[\"book\", \"idx_chapter\", \"idx_verse\"]] = idx_verse[1]\n",
    "#         nmv_embeddings = (\n",
    "#             nmv_embeddings\n",
    "#             .reset_index()\n",
    "#             .set_index([\"book\", \"idx_chapter\", \"idx_verse\"])\n",
    "#         )\n",
    "#         yield nmv_embeddings.drop(columns=\"embeddings\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from spacy.lang.fa import Persian\n",
    "from spacy.lang.en import English\n",
    "\n",
    "fa = Persian()\n",
    "eng = English()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "eng_word         [ב/ראשית, ברא, אלהים, את, ה/שמים, ו/את, ה/ארץ]\n",
      "strongs       [Hb/H7225, H1254, H430, H853, Hd/H8064, Hc/H85...\n",
      "morphology    [HR/Ncfsa, HVqp3ms, HNcmpa, HTo, HTd/Ncmpa, HC...\n",
      "Name: (Genesis, 0, 0), dtype: object\n",
      "word    [در, آغاز, ،, خدا, آسمانها, و, زمین, را, آفرید...\n",
      "Name: (Genesis, 0, 0), dtype: object\n"
     ]
    }
   ],
   "source": [
    "def GetWords(x):\n",
    "    return [a for sub in (s.split(\"/\") for s in list(x)) for a in sub]\n",
    "\n",
    "def ListWOFaPunctuation(x):\n",
    "    return[a for a in list(x) if not fa.vocab[a].is_punct and not fa.vocab[a].is_stop]\n",
    "\n",
    "nt = [\"Matthew\", \"Mark\", \"Luke\", \"John\", \"Acts\", \"Romans\", \"I Corinthians\", \"II Corinthians\", \"Galatians\", \"Ephesians\", \"Philippians\", \"Colossians\", \"I Thessalonians\", \"II Thessalonians\", \"I Timothy\", \"II Timothy\", \"Titus\", \"Philemon\", \"Hebrews\", \"James\", \"I Peter\", \"II Peter\", \"I John\", \"II John\", \"III John\", \"Jude\", \"Revelation of John\"]\n",
    "\n",
    "original = LoadBible(\"original\")\n",
    "original = (\n",
    "    original\n",
    "    .groupby([\"book\", \"idx_chapter\", \"idx_verse\"])\n",
    "    [[\"eng_word\", \"strongs\", \"morphology\"]]\n",
    "    .agg(list)\n",
    ")\n",
    "\n",
    "nmv = LoadBible(\"nmv_pos_tagged\")\n",
    "\n",
    "nmv = (\n",
    "    nmv\n",
    "    .groupby([\"book\", \"idx_chapter\", \"idx_verse\"])\n",
    "    [[\"word\"]]\n",
    "    .agg(list)\n",
    ")\n",
    "\n",
    "\n",
    "print(original.loc[(\"Genesis\",0,0)])\n",
    "print(nmv.loc[(\"Genesis\",0,0)])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ListWOEnPunctuation(x):\n",
    "    return[a for a in list(x) if not eng.vocab[\"None\" if a==None else a].is_punct]\n",
    "\n",
    "kjv = LoadBible(\"KJV\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "eng_word      [In the beginning, God, created, the heaven, a...\n",
      "strongs       [H7225, H430, H853 H1254, H8064, H853, H776, N...\n",
      "morphology         [None, None, TH8804, None, None, None, None]\n",
      "Name: (Genesis, 0, 0), dtype: object\n"
     ]
    }
   ],
   "source": [
    "kjv = (\n",
    "    kjv\n",
    "    .groupby([\"book\", \"idx_chapter\", \"idx_verse\"])\n",
    "    [[\"eng_word\", \"strongs\", \"morphology\"]]\n",
    "    .agg(list)\n",
    ")\n",
    "print(kjv.loc[(\"Genesis\",0,0)])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-multilingual-cased were not used when initializing BertModel: ['cls.seq_relationship.weight', 'cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "2022-11-03 10:37:17,443 - simalign.simalign - INFO - Initialized the EmbeddingLoader with model: bert-base-multilingual-cased\n"
     ]
    }
   ],
   "source": [
    "from simalign import SentenceAligner\n",
    "\n",
    "aligner = SentenceAligner()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"0\": \"G4413\", \"1\": \"G4413\", \"2\": \"G4413\", \"3\": \"G3056\", \"4\": \"G2321\", \"5\": \"G5599\", \"6\": \"G2321\", \"8\": \"G4160\"}\n",
      "[(0, 'G4413'), (1, 'G4413'), (2, 'G4413'), (3, 'G3056'), (4, None), (4, 'G2321'), (5, 'G5599'), (6, 'G2321'), (8, 'G4160'), (9, None)]\n",
      "{\"0\": \"G4413\", \"1\": \"G4413\", \"3\": \"G3056\", \"5\": \"G5599\", \"6\": \"G2321\", \"8\": \"G4160\"}\n",
      "[(0, 'G4413'), (1, 'G4413'), (3, 'G3056'), (4, None), (5, 'G5599'), (6, 'G2321'), (8, 'G4160'), (9, None)]\n",
      "{\"0\": \"G4413\", \"1\": \"G4413\", \"2\": \"G4413\", \"3\": \"G3056\", \"5\": \"G5599\", \"6\": \"G2321\", \"8\": \"G4160\"}\n",
      "[(0, 'G4413'), (1, 'G4413'), (2, 'G4413'), (3, 'G3056'), (4, None), (5, 'G5599'), (6, 'G2321'), (8, 'G4160'), (9, None)]\n"
     ]
    }
   ],
   "source": [
    "def cat(x):\n",
    "    return \" \".join(list(x.astype(str)))\n",
    "\n",
    "trg = kjv.loc[(\"Genesis\",0,0),\"eng_word\"]\n",
    "src = nmv.loc[(\"Genesis\",0,0),\"word\"]\n",
    "\n",
    "alignments = aligner.get_word_aligns(src, trg)\n",
    "\n",
    "for method in alignments:\n",
    "\n",
    "    method_alignments = [(srci, kjv.loc[(\"Acts\",0,0),\"strongs\"][trgi]) for srci,trgi in alignments[method]]\n",
    "    df = {key:value for key, value in pd.DataFrame.from_records(method_alignments).dropna().groupby(0).agg(cat).to_records().tolist()}\n",
    "\n",
    "    print(json.dumps(df))\n",
    "    print(method_alignments)\n",
    "    # print(method, \":\", [list(filter(lambda item: item is not None, [word, method_alignments[src.index(word)]])) if src.index(word) in method_alignments  else [word] for word in src])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "def cat(x):\n",
    "    return \" \".join(list(filter(lambda item: item is not None, x.astype(str))))\n",
    "\n",
    "def Align(row):\n",
    "    trg = row.eng_word\n",
    "    src = row.word\n",
    "    alignments = aligner.get_word_aligns(src, trg)\n",
    "    method_alignments = [(srci, row.strongs[trgi]) for srci,trgi in alignments[\"itermax\"]]\n",
    "    lookup = {key:value for key, value in pd.DataFrame.from_records(method_alignments).dropna().groupby(0).agg(cat).to_records().tolist()}\n",
    "    return [list(filter(lambda item: item is not None, [word, lookup[src.index(word)]])) if src.index(word) in lookup  else [word] for word in src]\n",
    "\n",
    "combined_df = kjv.join(nmv)\n",
    "combined_df.head()\n",
    "\n",
    "tqdm.pandas(desc=\"Aligning words for each verse in the Bible\")\n",
    "combined_df[\"fa_strongs\"] = combined_df.progress_apply(Align, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Aligning words for each verse in the Bible:  40%|████      | 2/5 [00:01<00:02,  1.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: 'G4160', 1: 'G3056', 2: 'G4413', 3: 'G3056', 6: 'G5599', 7: 'G2321', 9: 'G4413', 10: 'G4012', 12: 'G3956', 15: 'G3739', 16: 'G3588 G2424', 17: 'G4160', 18: 'G4160', 20: 'G2532', 21: 'G1321', 22: 'G5037', 23: 'G756'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Aligning words for each verse in the Bible:  60%|██████    | 3/5 [00:02<00:01,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: 'G891', 1: 'G2250', 2: 'G3739', 3: 'G353', 4: 'G353', 5: 'G4151 G40 G4151', 6: 'G1781', 7: 'G3588 G652', 8: 'G3588 G652', 9: 'G353 G1586', 10: 'G1586', 11: 'G1781', 13: 'G1781', 16: 'G1781'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Aligning words for each verse in the Bible:  80%|████████  | 4/5 [00:04<00:01,  1.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: 'G3936', 1: 'G3326', 4: 'G3936 G5039', 5: 'G846', 6: 'G5039', 8: 'G3739', 11: 'G3936', 13: 'G1722', 14: 'G5039', 15: 'G4183', 19: 'G2198', 25: 'G5062', 26: 'G2250', 27: 'G1223', 28: 'G846', 29: 'G3700', 31: 'G2532', 32: 'G3004 G4012', 33: 'G3588 G932', 34: 'G3588 G2316', 35: 'G4012', 36: 'G3588 G932'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Aligning words for each verse in the Bible: 100%|██████████| 5/5 [00:05<00:00,  1.38s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: 'G2532', 3: 'G4871', 4: 'G4871', 5: 'G3588 G3962', 6: 'added', 8: 'G3853', 9: 'G5563', 12: 'G575 G2414', 13: 'G5563', 14: 'G5563', 17: 'G235', 18: 'G4037', 19: 'G3588 G1860', 20: 'G3588 G1860', 21: 'G3588 G3962', 23: 'G3739', 24: 'G3450', 25: 'G3450', 26: 'added G191'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Aligning words for each verse in the Bible: 100%|██████████| 5/5 [00:07<00:00,  1.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: 'G3754', 1: 'G2491 G3303', 2: 'G5204', 3: 'G5204', 4: 'G907', 7: 'G1161', 8: 'G4183', 9: 'G2250 G3326', 11: 'G907', 13: 'G5210', 14: 'G1722', 15: 'G4151 G40 G4151', 16: 'G907', 17: 'G907'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "book  idx_chapter  idx_verse\n",
       "Acts  0            0            [[من, G4160], [کتاب, G3056], [نخست, G4413], [خ...\n",
       "                   1            [[تا, G891], [روزی, G2250], [که, G3739], [به, ...\n",
       "                   2            [[او, G3936], [پس, G3326], [از], [رنج], [کشیدن...\n",
       "                   3            [[یک, G2532], [بار], [در], [حین, G4871], [صرف,...\n",
       "                   4            [[زیرا, G3754], [یحیی, G2491 G3303], [با, G520...\n",
       "dtype: object"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined_df.head().progress_apply(Align, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "word    [در, آغاز, ،, خدا, آسمانها, و, زمین, را, آفرید...\n",
      "Name: (Genesis, 0, 0), dtype: object\n"
     ]
    }
   ],
   "source": [
    "print(trg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "OSError",
     "evalue": "Error no file named pytorch_model.bin, tf_model.h5, model.ckpt.index or flax_model.msgpack found in directory C:\\Users\\saaam\\farsi-strongs\\outputs.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[1;31mOSError\u001b[0m                                   Traceback (most recent call last)\n",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_464/293160510.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n",
      "\u001b[0;32m     24\u001b[0m \u001b[1;31m# fa_pos_tagger = POSTagger(\"model/perpos.model\")\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m     25\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m---> 26\u001b[1;33m \u001b[0mconfig\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mActivateModel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodelname\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0menviron\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"USERPROFILE\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"farsi-strongs\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"outputs\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_464/2209945142.py\u001b[0m in \u001b[0;36mActivateModel\u001b[1;34m(modelname)\u001b[0m\n",
      "\u001b[0;32m     19\u001b[0m     \u001b[0mconfig\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mAutoConfig\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodelname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m     20\u001b[0m     \u001b[0mtokenizer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mAutoTokenizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"bert-base-multilingual-cased\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m---> 21\u001b[1;33m     \u001b[0mmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mAutoModel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodelname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m     22\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mconfig\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m     23\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[1;32mc:\\Users\\saaam\\venv310\\lib\\site-packages\\transformers\\models\\auto\\auto_factory.py\u001b[0m in \u001b[0;36mfrom_pretrained\u001b[1;34m(cls, pretrained_model_name_or_path, *model_args, **kwargs)\u001b[0m\n",
      "\u001b[0;32m    444\u001b[0m         \u001b[1;32melif\u001b[0m \u001b[0mtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mcls\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_model_mapping\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m    445\u001b[0m             \u001b[0mmodel_class\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_get_model_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcls\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_model_mapping\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m--> 446\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mmodel_class\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpretrained_model_name_or_path\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0mmodel_args\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m    447\u001b[0m         raise ValueError(\n",
      "\u001b[0;32m    448\u001b[0m             \u001b[1;34mf\"Unrecognized configuration class {config.__class__} for this kind of AutoModel: {cls.__name__}.\\n\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[1;32mc:\\Users\\saaam\\venv310\\lib\\site-packages\\transformers\\modeling_utils.py\u001b[0m in \u001b[0;36mfrom_pretrained\u001b[1;34m(cls, pretrained_model_name_or_path, *model_args, **kwargs)\u001b[0m\n",
      "\u001b[0;32m   1977\u001b[0m                     )\n",
      "\u001b[0;32m   1978\u001b[0m                 \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m-> 1979\u001b[1;33m                     raise EnvironmentError(\n",
      "\u001b[0m\u001b[0;32m   1980\u001b[0m                         \u001b[1;34mf\"Error no file named {WEIGHTS_NAME}, {TF2_WEIGHTS_NAME}, {TF_WEIGHTS_NAME + '.index'} or \"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m   1981\u001b[0m                         \u001b[1;34mf\"{FLAX_WEIGHTS_NAME} found in directory {pretrained_model_name_or_path}.\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[1;31mOSError\u001b[0m: Error no file named pytorch_model.bin, tf_model.h5, model.ckpt.index or flax_model.msgpack found in directory C:\\Users\\saaam\\farsi-strongs\\outputs."
     ]
    }
   ],
   "source": [
    "nt = [\"Matthew\", \"Mark\", \"Luke\", \"John\", \"Acts\", \"Romans\", \"I Corinthians\", \"II Corinthians\", \"Galatians\", \"Ephesians\", \"Philippians\", \"Colossians\", \"I Thessalonians\", \"II Thessalonians\", \"I Timothy\", \"II Timothy\", \"Titus\", \"Philemon\", \"Hebrews\", \"James\", \"I Peter\", \"II Peter\", \"I John\", \"II John\", \"III John\", \"Jude\", \"Revelation of John\"]\n",
    "\n",
    "original = LoadBible(\"original\")\n",
    "original_ot = (\n",
    "    original\n",
    "    .loc[~original.book.isin(nt)]\n",
    "    .set_index([\"book\", \"idx_chapter\", \"idx_verse\", \"idx_word\"])\n",
    "    [[\"eng_word\", \"strongs\", \"morphology\"]]\n",
    ")\n",
    "\n",
    "nmv = LoadBible(\"nmv_pos_tagged\")\n",
    "\n",
    "nmv_ot = (\n",
    "    nmv\n",
    "    .loc[~nmv.book.isin(nt)]\n",
    "    .set_index([\"book\", \"idx_chapter\", \"idx_verse\", \"idx_word\"])\n",
    "    [[\"word\", \"pos_tag\"]]\n",
    ")\n",
    "\n",
    "fa_nlp = Persian()\n",
    "he_nlp = Hebrew()\n",
    "gr_nlp = Greek()\n",
    "\n",
    "# fa_pos_tagger = POSTagger(\"model/perpos.model\")\n",
    "\n",
    "config, tokenizer, model = ActivateModel(modelname=os.path.join(os.environ[\"USERPROFILE\"], \"farsi-strongs\", \"outputs\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "POS tagging farsi words: 100%|██████████| 31103/31103 [01:46<00:00, 291.51it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>idx_word</th>\n",
       "      <th>word</th>\n",
       "      <th>pos_tag</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>book</th>\n",
       "      <th>idx_chapter</th>\n",
       "      <th>idx_verse</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">Acts</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">0</th>\n",
       "      <th>0</th>\n",
       "      <td>[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,...</td>\n",
       "      <td>[من, کتاب, نخست, خود, را, ،, ای, تِئوفیلوس, ،,...</td>\n",
       "      <td>[(من, RES), (کتاب, RES), (نخست, RES), (خود, RE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,...</td>\n",
       "      <td>[تا, روزی, که, به, واسطۀ, روح‌القدس, دستورهایی...</td>\n",
       "      <td>[(تا, CONJ), (روزی, N), (که, CONJ), (به, P), (...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,...</td>\n",
       "      <td>[او, پس, از, رنج, کشیدن, ،, خویشتن, را, بر, آن...</td>\n",
       "      <td>[(او, PRO), (پس, P), (از, P), (رنج, Ne), (کشید...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,...</td>\n",
       "      <td>[یک, بار, در, حین, صرف, غذا, بدیشان, امر, فرمو...</td>\n",
       "      <td>[(یک, NUM), (بار, N), (در, P), (حین, Ne), (صرف...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,...</td>\n",
       "      <td>[زیرا, یحیی, با, آب, تعمید, می‌داد, ،, امّا, چ...</td>\n",
       "      <td>[(زیرا, CONJ), (یحیی, N), (با, P), (آب, N), (ت...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                     idx_word  \\\n",
       "book idx_chapter idx_verse                                                      \n",
       "Acts 0           0          [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,...   \n",
       "                 1          [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,...   \n",
       "                 2          [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,...   \n",
       "                 3          [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,...   \n",
       "                 4          [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,...   \n",
       "\n",
       "                                                                         word  \\\n",
       "book idx_chapter idx_verse                                                      \n",
       "Acts 0           0          [من, کتاب, نخست, خود, را, ،, ای, تِئوفیلوس, ،,...   \n",
       "                 1          [تا, روزی, که, به, واسطۀ, روح‌القدس, دستورهایی...   \n",
       "                 2          [او, پس, از, رنج, کشیدن, ،, خویشتن, را, بر, آن...   \n",
       "                 3          [یک, بار, در, حین, صرف, غذا, بدیشان, امر, فرمو...   \n",
       "                 4          [زیرا, یحیی, با, آب, تعمید, می‌داد, ،, امّا, چ...   \n",
       "\n",
       "                                                                      pos_tag  \n",
       "book idx_chapter idx_verse                                                     \n",
       "Acts 0           0          [(من, RES), (کتاب, RES), (نخست, RES), (خود, RE...  \n",
       "                 1          [(تا, CONJ), (روزی, N), (که, CONJ), (به, P), (...  \n",
       "                 2          [(او, PRO), (پس, P), (از, P), (رنج, Ne), (کشید...  \n",
       "                 3          [(یک, NUM), (بار, N), (در, P), (حین, Ne), (صرف...  \n",
       "                 4          [(زیرا, CONJ), (یحیی, N), (با, P), (آب, N), (ت...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# tqdm.pandas(desc=\"POS tagging farsi words\")\n",
    "# nmv[\"pos_tag\"] = nmv.word.progress_apply(fa_pos_tagger.parse)\n",
    "# nmv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "original[\"exclude\"] = original.eng_word.apply(\n",
    "    lambda x:any(\n",
    "        [\n",
    "            he_nlp.vocab[x].is_stop, \n",
    "            he_nlp.vocab[x].is_punct, \n",
    "            gr_nlp.vocab[x].is_stop, \n",
    "            gr_nlp.vocab[x].is_punct,\n",
    "        ]\n",
    "    )\n",
    ")\n",
    "\n",
    "nmv[\"exclude\"] = nmv.word.apply(lambda x:any([fa_nlp.vocab[x].is_stop, fa_nlp.vocab[x].is_punct]))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "original_subset = (\n",
    "    original\n",
    "    .loc[~original.exclude]\n",
    "    .reset_index()\n",
    "    .groupby([\"book\", \"idx_chapter\", \"idx_verse\"])\n",
    "    [[\"eng_word\", \"strongs\", \"morphology\", \"idx_word\"]]\n",
    "    .agg(list)\n",
    ")\n",
    "\n",
    "nmv_subset = (\n",
    "    nmv\n",
    "    .loc[~nmv.exclude]\n",
    "    .reset_index()\n",
    "    .groupby([\"book\", \"idx_chapter\", \"idx_verse\"])\n",
    "    [[\"word\", \"pos_tag\", \"idx_word\"]]\n",
    "    .agg(list)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_df = (\n",
    "    original_subset\n",
    "    .join(\n",
    "        nmv_subset,\n",
    "        how=\"left\",\n",
    "        lsuffix=\"_original\",\n",
    "        rsuffix=\"_nmv\"\n",
    "    )\n",
    ").loc[(\"Genesis\",0,0):(\"Genesis\",0,1),:]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>eng_word</th>\n",
       "      <th>strongs</th>\n",
       "      <th>morphology</th>\n",
       "      <th>idx_word_original</th>\n",
       "      <th>word</th>\n",
       "      <th>pos_tag</th>\n",
       "      <th>idx_word_nmv</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>book</th>\n",
       "      <th>idx_chapter</th>\n",
       "      <th>idx_verse</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Genesis</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">0</th>\n",
       "      <th>0</th>\n",
       "      <td>[ב/ראשית, ברא, אלהים, ה/שמים, ו/את, ה/ארץ]</td>\n",
       "      <td>[Hb/H7225, H1254, H430, Hd/H8064, Hc/H853, Hd/...</td>\n",
       "      <td>[HR/Ncfsa, HVqp3ms, HNcmpa, HTd/Ncmpa, HC/To, ...</td>\n",
       "      <td>[0, 1, 2, 4, 5, 6]</td>\n",
       "      <td>[آغاز, خدا, آسمانها, زمین, آفرید]</td>\n",
       "      <td>[[آغاز, N], [خدا, N], [آسمانها, N], [زمین, N],...</td>\n",
       "      <td>[1, 3, 4, 6, 8]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[ו/ה/ארץ, תהו, ו/בהו, ו/חשך, פני, תהום, ו/רוח,...</td>\n",
       "      <td>[Hc/Hd/H776, H8414, Hc/H922, Hc/H2822, H6440, ...</td>\n",
       "      <td>[HC/Td/Ncbsa, HNcmsa, HC/Ncmsa, HC/Ncmsa, HNcb...</td>\n",
       "      <td>[0, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13]</td>\n",
       "      <td>[زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...</td>\n",
       "      <td>[[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...</td>\n",
       "      <td>[0, 1, 3, 7, 10, 13, 14, 15, 16, 18]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                        eng_word  \\\n",
       "book    idx_chapter idx_verse                                                      \n",
       "Genesis 0           0                 [ב/ראשית, ברא, אלהים, ה/שמים, ו/את, ה/ארץ]   \n",
       "                    1          [ו/ה/ארץ, תהו, ו/בהו, ו/חשך, פני, תהום, ו/רוח,...   \n",
       "\n",
       "                                                                         strongs  \\\n",
       "book    idx_chapter idx_verse                                                      \n",
       "Genesis 0           0          [Hb/H7225, H1254, H430, Hd/H8064, Hc/H853, Hd/...   \n",
       "                    1          [Hc/Hd/H776, H8414, Hc/H922, Hc/H2822, H6440, ...   \n",
       "\n",
       "                                                                      morphology  \\\n",
       "book    idx_chapter idx_verse                                                      \n",
       "Genesis 0           0          [HR/Ncfsa, HVqp3ms, HNcmpa, HTd/Ncmpa, HC/To, ...   \n",
       "                    1          [HC/Td/Ncbsa, HNcmsa, HC/Ncmsa, HC/Ncmsa, HNcb...   \n",
       "\n",
       "                                                  idx_word_original  \\\n",
       "book    idx_chapter idx_verse                                         \n",
       "Genesis 0           0                            [0, 1, 2, 4, 5, 6]   \n",
       "                    1          [0, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13]   \n",
       "\n",
       "                                                                            word  \\\n",
       "book    idx_chapter idx_verse                                                      \n",
       "Genesis 0           0                          [آغاز, خدا, آسمانها, زمین, آفرید]   \n",
       "                    1          [زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...   \n",
       "\n",
       "                                                                         pos_tag  \\\n",
       "book    idx_chapter idx_verse                                                      \n",
       "Genesis 0           0          [[آغاز, N], [خدا, N], [آسمانها, N], [زمین, N],...   \n",
       "                    1          [[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...   \n",
       "\n",
       "                                                       idx_word_nmv  \n",
       "book    idx_chapter idx_verse                                        \n",
       "Genesis 0           0                               [1, 3, 4, 6, 8]  \n",
       "                    1          [0, 1, 3, 7, 10, 13, 14, 15, 16, 18]  "
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "matching words: 100%|██████████| 2/2 [00:00<00:00,  3.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx_wordmatch_nmv                                   [3, 3, 3, 3, 3, 3]\n",
      "wordmatch_nmv                           [خدا, خدا, خدا, خدا, خدا, خدا]\n",
      "pos_tag_nmv          [[خدا, N], [خدا, N], [خدا, N], [خدا, N], [خدا,...\n",
      "cosine_similarity    [[[0.5750017952637303]], [[0.5802883873468905]...\n",
      "Name: 0, dtype: object\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "matching words: 100%|██████████| 2/2 [00:01<00:00,  1.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx_wordmatch_nmv          [14, 14, 14, 14, 14, 14, 0, 14, 14, 14, 14]\n",
      "wordmatch_nmv        [خدا, خدا, خدا, خدا, خدا, خدا, زمین, خدا, خدا,...\n",
      "pos_tag_nmv          [[خدا, N], [خدا, N], [خدا, N], [خدا, N], [خدا,...\n",
      "cosine_similarity    [[[0.5539344012703646]], [[0.569367715437644]]...\n",
      "Name: 0, dtype: object\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>eng_word</th>\n",
       "      <th>strongs</th>\n",
       "      <th>morphology</th>\n",
       "      <th>idx_word_original</th>\n",
       "      <th>word</th>\n",
       "      <th>pos_tag</th>\n",
       "      <th>idx_word_nmv</th>\n",
       "      <th>idx_wordmatch_nmv</th>\n",
       "      <th>wordmatch_nmv</th>\n",
       "      <th>pos_tag_nmv</th>\n",
       "      <th>cosine_similarity</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>book</th>\n",
       "      <th>idx_chapter</th>\n",
       "      <th>idx_verse</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Genesis</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">0</th>\n",
       "      <th>0</th>\n",
       "      <td>[ב/ראשית, ברא, אלהים, ה/שמים, ו/את, ה/ארץ]</td>\n",
       "      <td>[Hb/H7225, H1254, H430, Hd/H8064, Hc/H853, Hd/...</td>\n",
       "      <td>[HR/Ncfsa, HVqp3ms, HNcmpa, HTd/Ncmpa, HC/To, ...</td>\n",
       "      <td>[0, 1, 2, 4, 5, 6]</td>\n",
       "      <td>[آغاز, خدا, آسمانها, زمین, آفرید]</td>\n",
       "      <td>[[آغاز, N], [خدا, N], [آسمانها, N], [زمین, N],...</td>\n",
       "      <td>[1, 3, 4, 6, 8]</td>\n",
       "      <td>[3, 3, 3, 3, 3, 3]</td>\n",
       "      <td>[خدا, خدا, خدا, خدا, خدا, خدا]</td>\n",
       "      <td>[[خدا, N], [خدا, N], [خدا, N], [خدا, N], [خدا,...</td>\n",
       "      <td>[[[0.5750017952637303]], [[0.5802883873468905]...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[ו/ה/ארץ, תהו, ו/בהו, ו/חשך, פני, תהום, ו/רוח,...</td>\n",
       "      <td>[Hc/Hd/H776, H8414, Hc/H922, Hc/H2822, H6440, ...</td>\n",
       "      <td>[HC/Td/Ncbsa, HNcmsa, HC/Ncmsa, HC/Ncmsa, HNcb...</td>\n",
       "      <td>[0, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13]</td>\n",
       "      <td>[زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...</td>\n",
       "      <td>[[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...</td>\n",
       "      <td>[0, 1, 3, 7, 10, 13, 14, 15, 16, 18]</td>\n",
       "      <td>[14, 14, 14, 14, 14, 14, 0, 14, 14, 14, 14]</td>\n",
       "      <td>[خدا, خدا, خدا, خدا, خدا, خدا, زمین, خدا, خدا,...</td>\n",
       "      <td>[[خدا, N], [خدا, N], [خدا, N], [خدا, N], [خدا,...</td>\n",
       "      <td>[[[0.5539344012703646]], [[0.569367715437644]]...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                        eng_word  \\\n",
       "book    idx_chapter idx_verse                                                      \n",
       "Genesis 0           0                 [ב/ראשית, ברא, אלהים, ה/שמים, ו/את, ה/ארץ]   \n",
       "                    1          [ו/ה/ארץ, תהו, ו/בהו, ו/חשך, פני, תהום, ו/רוח,...   \n",
       "\n",
       "                                                                         strongs  \\\n",
       "book    idx_chapter idx_verse                                                      \n",
       "Genesis 0           0          [Hb/H7225, H1254, H430, Hd/H8064, Hc/H853, Hd/...   \n",
       "                    1          [Hc/Hd/H776, H8414, Hc/H922, Hc/H2822, H6440, ...   \n",
       "\n",
       "                                                                      morphology  \\\n",
       "book    idx_chapter idx_verse                                                      \n",
       "Genesis 0           0          [HR/Ncfsa, HVqp3ms, HNcmpa, HTd/Ncmpa, HC/To, ...   \n",
       "                    1          [HC/Td/Ncbsa, HNcmsa, HC/Ncmsa, HC/Ncmsa, HNcb...   \n",
       "\n",
       "                                                  idx_word_original  \\\n",
       "book    idx_chapter idx_verse                                         \n",
       "Genesis 0           0                            [0, 1, 2, 4, 5, 6]   \n",
       "                    1          [0, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13]   \n",
       "\n",
       "                                                                            word  \\\n",
       "book    idx_chapter idx_verse                                                      \n",
       "Genesis 0           0                          [آغاز, خدا, آسمانها, زمین, آفرید]   \n",
       "                    1          [زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...   \n",
       "\n",
       "                                                                         pos_tag  \\\n",
       "book    idx_chapter idx_verse                                                      \n",
       "Genesis 0           0          [[آغاز, N], [خدا, N], [آسمانها, N], [زمین, N],...   \n",
       "                    1          [[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...   \n",
       "\n",
       "                                                       idx_word_nmv  \\\n",
       "book    idx_chapter idx_verse                                         \n",
       "Genesis 0           0                               [1, 3, 4, 6, 8]   \n",
       "                    1          [0, 1, 3, 7, 10, 13, 14, 15, 16, 18]   \n",
       "\n",
       "                                                         idx_wordmatch_nmv  \\\n",
       "book    idx_chapter idx_verse                                                \n",
       "Genesis 0           0                                   [3, 3, 3, 3, 3, 3]   \n",
       "                    1          [14, 14, 14, 14, 14, 14, 0, 14, 14, 14, 14]   \n",
       "\n",
       "                                                                   wordmatch_nmv  \\\n",
       "book    idx_chapter idx_verse                                                      \n",
       "Genesis 0           0                             [خدا, خدا, خدا, خدا, خدا, خدا]   \n",
       "                    1          [خدا, خدا, خدا, خدا, خدا, خدا, زمین, خدا, خدا,...   \n",
       "\n",
       "                                                                     pos_tag_nmv  \\\n",
       "book    idx_chapter idx_verse                                                      \n",
       "Genesis 0           0          [[خدا, N], [خدا, N], [خدا, N], [خدا, N], [خدا,...   \n",
       "                    1          [[خدا, N], [خدا, N], [خدا, N], [خدا, N], [خدا,...   \n",
       "\n",
       "                                                               cosine_similarity  \n",
       "book    idx_chapter idx_verse                                                     \n",
       "Genesis 0           0          [[[0.5750017952637303]], [[0.5802883873468905]...  \n",
       "                    1          [[[0.5539344012703646]], [[0.569367715437644]]...  "
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "tqdm.pandas(desc=\"matching words\")\n",
    "merged_df[[\"idx_wordmatch_nmv\", \"wordmatch_nmv\", \"pos_tag_nmv\", \"cosine_similarity\"]] = (\n",
    "    merged_df\n",
    "    .progress_apply(\n",
    "        max_cos_sim, \n",
    "        args=(model, tokenizer), \n",
    "        axis=1\n",
    "    )\n",
    ")\n",
    "\n",
    "merged_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>eng_word</th>\n",
       "      <th>strongs</th>\n",
       "      <th>morphology</th>\n",
       "      <th>idx_word_original</th>\n",
       "      <th>word</th>\n",
       "      <th>pos_tag</th>\n",
       "      <th>idx_word_nmv</th>\n",
       "      <th>idx_wordmatch_nmv</th>\n",
       "      <th>wordmatch_nmv</th>\n",
       "      <th>pos_tag_nmv</th>\n",
       "      <th>cosine_similarity</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>book</th>\n",
       "      <th>idx_chapter</th>\n",
       "      <th>idx_verse</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"17\" valign=\"top\">Genesis</th>\n",
       "      <th rowspan=\"17\" valign=\"top\">0</th>\n",
       "      <th>0</th>\n",
       "      <td>ב/ראשית</td>\n",
       "      <td>Hb/H7225</td>\n",
       "      <td>HR/Ncfsa</td>\n",
       "      <td>0</td>\n",
       "      <td>[آغاز, خدا, آسمانها, زمین, آفرید]</td>\n",
       "      <td>[[آغاز, N], [خدا, N], [آسمانها, N], [زمین, N],...</td>\n",
       "      <td>[1, 3, 4, 6, 8]</td>\n",
       "      <td>3</td>\n",
       "      <td>خدا</td>\n",
       "      <td>[خدا, N]</td>\n",
       "      <td>[[0.5750017952637303]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ברא</td>\n",
       "      <td>H1254</td>\n",
       "      <td>HVqp3ms</td>\n",
       "      <td>1</td>\n",
       "      <td>[آغاز, خدا, آسمانها, زمین, آفرید]</td>\n",
       "      <td>[[آغاز, N], [خدا, N], [آسمانها, N], [زمین, N],...</td>\n",
       "      <td>[1, 3, 4, 6, 8]</td>\n",
       "      <td>3</td>\n",
       "      <td>خدا</td>\n",
       "      <td>[خدا, N]</td>\n",
       "      <td>[[0.5802883873468905]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>אלהים</td>\n",
       "      <td>H430</td>\n",
       "      <td>HNcmpa</td>\n",
       "      <td>2</td>\n",
       "      <td>[آغاز, خدا, آسمانها, زمین, آفرید]</td>\n",
       "      <td>[[آغاز, N], [خدا, N], [آسمانها, N], [زمین, N],...</td>\n",
       "      <td>[1, 3, 4, 6, 8]</td>\n",
       "      <td>3</td>\n",
       "      <td>خدا</td>\n",
       "      <td>[خدا, N]</td>\n",
       "      <td>[[0.5861434809310097]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ה/שמים</td>\n",
       "      <td>Hd/H8064</td>\n",
       "      <td>HTd/Ncmpa</td>\n",
       "      <td>4</td>\n",
       "      <td>[آغاز, خدا, آسمانها, زمین, آفرید]</td>\n",
       "      <td>[[آغاز, N], [خدا, N], [آسمانها, N], [زمین, N],...</td>\n",
       "      <td>[1, 3, 4, 6, 8]</td>\n",
       "      <td>3</td>\n",
       "      <td>خدا</td>\n",
       "      <td>[خدا, N]</td>\n",
       "      <td>[[0.6244082085960594]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ו/את</td>\n",
       "      <td>Hc/H853</td>\n",
       "      <td>HC/To</td>\n",
       "      <td>5</td>\n",
       "      <td>[آغاز, خدا, آسمانها, زمین, آفرید]</td>\n",
       "      <td>[[آغاز, N], [خدا, N], [آسمانها, N], [زمین, N],...</td>\n",
       "      <td>[1, 3, 4, 6, 8]</td>\n",
       "      <td>3</td>\n",
       "      <td>خدا</td>\n",
       "      <td>[خدا, N]</td>\n",
       "      <td>[[0.6315576137669605]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ה/ארץ</td>\n",
       "      <td>Hd/H776</td>\n",
       "      <td>HTd/Ncbsa</td>\n",
       "      <td>6</td>\n",
       "      <td>[آغاز, خدا, آسمانها, زمین, آفرید]</td>\n",
       "      <td>[[آغاز, N], [خدا, N], [آسمانها, N], [زمین, N],...</td>\n",
       "      <td>[1, 3, 4, 6, 8]</td>\n",
       "      <td>3</td>\n",
       "      <td>خدا</td>\n",
       "      <td>[خدا, N]</td>\n",
       "      <td>[[0.6730095257915146]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ו/ה/ארץ</td>\n",
       "      <td>Hc/Hd/H776</td>\n",
       "      <td>HC/Td/Ncbsa</td>\n",
       "      <td>0</td>\n",
       "      <td>[زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...</td>\n",
       "      <td>[[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...</td>\n",
       "      <td>[0, 1, 3, 7, 10, 13, 14, 15, 16, 18]</td>\n",
       "      <td>14</td>\n",
       "      <td>خدا</td>\n",
       "      <td>[خدا, N]</td>\n",
       "      <td>[[0.5539344012703646]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>תהו</td>\n",
       "      <td>H8414</td>\n",
       "      <td>HNcmsa</td>\n",
       "      <td>2</td>\n",
       "      <td>[زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...</td>\n",
       "      <td>[[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...</td>\n",
       "      <td>[0, 1, 3, 7, 10, 13, 14, 15, 16, 18]</td>\n",
       "      <td>14</td>\n",
       "      <td>خدا</td>\n",
       "      <td>[خدا, N]</td>\n",
       "      <td>[[0.569367715437644]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ו/בהו</td>\n",
       "      <td>Hc/H922</td>\n",
       "      <td>HC/Ncmsa</td>\n",
       "      <td>3</td>\n",
       "      <td>[زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...</td>\n",
       "      <td>[[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...</td>\n",
       "      <td>[0, 1, 3, 7, 10, 13, 14, 15, 16, 18]</td>\n",
       "      <td>14</td>\n",
       "      <td>خدا</td>\n",
       "      <td>[خدا, N]</td>\n",
       "      <td>[[0.5956766805237825]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ו/חשך</td>\n",
       "      <td>Hc/H2822</td>\n",
       "      <td>HC/Ncmsa</td>\n",
       "      <td>4</td>\n",
       "      <td>[زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...</td>\n",
       "      <td>[[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...</td>\n",
       "      <td>[0, 1, 3, 7, 10, 13, 14, 15, 16, 18]</td>\n",
       "      <td>14</td>\n",
       "      <td>خدا</td>\n",
       "      <td>[خدا, N]</td>\n",
       "      <td>[[0.6016001426782509]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>פני</td>\n",
       "      <td>H6440</td>\n",
       "      <td>HNcbpc</td>\n",
       "      <td>6</td>\n",
       "      <td>[زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...</td>\n",
       "      <td>[[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...</td>\n",
       "      <td>[0, 1, 3, 7, 10, 13, 14, 15, 16, 18]</td>\n",
       "      <td>14</td>\n",
       "      <td>خدا</td>\n",
       "      <td>[خدا, N]</td>\n",
       "      <td>[[0.6606292771967468]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>תהום</td>\n",
       "      <td>H8415</td>\n",
       "      <td>HNcbsa</td>\n",
       "      <td>7</td>\n",
       "      <td>[زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...</td>\n",
       "      <td>[[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...</td>\n",
       "      <td>[0, 1, 3, 7, 10, 13, 14, 15, 16, 18]</td>\n",
       "      <td>14</td>\n",
       "      <td>خدا</td>\n",
       "      <td>[خدا, N]</td>\n",
       "      <td>[[0.6650169715287683]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ו/רוח</td>\n",
       "      <td>Hc/H7307</td>\n",
       "      <td>HC/Ncbsc</td>\n",
       "      <td>8</td>\n",
       "      <td>[زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...</td>\n",
       "      <td>[[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...</td>\n",
       "      <td>[0, 1, 3, 7, 10, 13, 14, 15, 16, 18]</td>\n",
       "      <td>0</td>\n",
       "      <td>زمین</td>\n",
       "      <td>[زمین, Ne]</td>\n",
       "      <td>[[0.6706229051016932]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>אלהים</td>\n",
       "      <td>H430</td>\n",
       "      <td>HNcmpa</td>\n",
       "      <td>9</td>\n",
       "      <td>[زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...</td>\n",
       "      <td>[[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...</td>\n",
       "      <td>[0, 1, 3, 7, 10, 13, 14, 15, 16, 18]</td>\n",
       "      <td>14</td>\n",
       "      <td>خدا</td>\n",
       "      <td>[خدا, N]</td>\n",
       "      <td>[[0.6730095636119798]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>מרחפת</td>\n",
       "      <td>H7363</td>\n",
       "      <td>HVprfsa</td>\n",
       "      <td>10</td>\n",
       "      <td>[زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...</td>\n",
       "      <td>[[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...</td>\n",
       "      <td>[0, 1, 3, 7, 10, 13, 14, 15, 16, 18]</td>\n",
       "      <td>14</td>\n",
       "      <td>خدا</td>\n",
       "      <td>[خدا, N]</td>\n",
       "      <td>[[0.6836679391320224]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>פני</td>\n",
       "      <td>H6440</td>\n",
       "      <td>HNcbpc</td>\n",
       "      <td>12</td>\n",
       "      <td>[زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...</td>\n",
       "      <td>[[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...</td>\n",
       "      <td>[0, 1, 3, 7, 10, 13, 14, 15, 16, 18]</td>\n",
       "      <td>14</td>\n",
       "      <td>خدا</td>\n",
       "      <td>[خدا, N]</td>\n",
       "      <td>[[0.7052896188538254]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ה/מים</td>\n",
       "      <td>Hd/H4325</td>\n",
       "      <td>HTd/Ncmpa</td>\n",
       "      <td>13</td>\n",
       "      <td>[زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...</td>\n",
       "      <td>[[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...</td>\n",
       "      <td>[0, 1, 3, 7, 10, 13, 14, 15, 16, 18]</td>\n",
       "      <td>14</td>\n",
       "      <td>خدا</td>\n",
       "      <td>[خدا, N]</td>\n",
       "      <td>[[0.7052896188538254]]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              eng_word     strongs   morphology  \\\n",
       "book    idx_chapter idx_verse                                     \n",
       "Genesis 0           0          ב/ראשית    Hb/H7225     HR/Ncfsa   \n",
       "                    0              ברא       H1254      HVqp3ms   \n",
       "                    0            אלהים        H430       HNcmpa   \n",
       "                    0           ה/שמים    Hd/H8064    HTd/Ncmpa   \n",
       "                    0             ו/את     Hc/H853        HC/To   \n",
       "                    0            ה/ארץ     Hd/H776    HTd/Ncbsa   \n",
       "                    1          ו/ה/ארץ  Hc/Hd/H776  HC/Td/Ncbsa   \n",
       "                    1              תהו       H8414       HNcmsa   \n",
       "                    1            ו/בהו     Hc/H922     HC/Ncmsa   \n",
       "                    1            ו/חשך    Hc/H2822     HC/Ncmsa   \n",
       "                    1              פני       H6440       HNcbpc   \n",
       "                    1             תהום       H8415       HNcbsa   \n",
       "                    1            ו/רוח    Hc/H7307     HC/Ncbsc   \n",
       "                    1            אלהים        H430       HNcmpa   \n",
       "                    1            מרחפת       H7363      HVprfsa   \n",
       "                    1              פני       H6440       HNcbpc   \n",
       "                    1            ה/מים    Hd/H4325    HTd/Ncmpa   \n",
       "\n",
       "                              idx_word_original  \\\n",
       "book    idx_chapter idx_verse                     \n",
       "Genesis 0           0                         0   \n",
       "                    0                         1   \n",
       "                    0                         2   \n",
       "                    0                         4   \n",
       "                    0                         5   \n",
       "                    0                         6   \n",
       "                    1                         0   \n",
       "                    1                         2   \n",
       "                    1                         3   \n",
       "                    1                         4   \n",
       "                    1                         6   \n",
       "                    1                         7   \n",
       "                    1                         8   \n",
       "                    1                         9   \n",
       "                    1                        10   \n",
       "                    1                        12   \n",
       "                    1                        13   \n",
       "\n",
       "                                                                            word  \\\n",
       "book    idx_chapter idx_verse                                                      \n",
       "Genesis 0           0                          [آغاز, خدا, آسمانها, زمین, آفرید]   \n",
       "                    0                          [آغاز, خدا, آسمانها, زمین, آفرید]   \n",
       "                    0                          [آغاز, خدا, آسمانها, زمین, آفرید]   \n",
       "                    0                          [آغاز, خدا, آسمانها, زمین, آفرید]   \n",
       "                    0                          [آغاز, خدا, آسمانها, زمین, آفرید]   \n",
       "                    0                          [آغاز, خدا, آسمانها, زمین, آفرید]   \n",
       "                    1          [زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...   \n",
       "                    1          [زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...   \n",
       "                    1          [زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...   \n",
       "                    1          [زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...   \n",
       "                    1          [زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...   \n",
       "                    1          [زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...   \n",
       "                    1          [زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...   \n",
       "                    1          [زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...   \n",
       "                    1          [زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...   \n",
       "                    1          [زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...   \n",
       "                    1          [زمین, بی‌شکل, خالی, تاریکی, ژرفا, روح, خدا, س...   \n",
       "\n",
       "                                                                         pos_tag  \\\n",
       "book    idx_chapter idx_verse                                                      \n",
       "Genesis 0           0          [[آغاز, N], [خدا, N], [آسمانها, N], [زمین, N],...   \n",
       "                    0          [[آغاز, N], [خدا, N], [آسمانها, N], [زمین, N],...   \n",
       "                    0          [[آغاز, N], [خدا, N], [آسمانها, N], [زمین, N],...   \n",
       "                    0          [[آغاز, N], [خدا, N], [آسمانها, N], [زمین, N],...   \n",
       "                    0          [[آغاز, N], [خدا, N], [آسمانها, N], [زمین, N],...   \n",
       "                    0          [[آغاز, N], [خدا, N], [آسمانها, N], [زمین, N],...   \n",
       "                    1          [[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...   \n",
       "                    1          [[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...   \n",
       "                    1          [[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...   \n",
       "                    1          [[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...   \n",
       "                    1          [[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...   \n",
       "                    1          [[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...   \n",
       "                    1          [[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...   \n",
       "                    1          [[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...   \n",
       "                    1          [[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...   \n",
       "                    1          [[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...   \n",
       "                    1          [[زمین, Ne], [بی‌شکل, AJ], [خالی, AJ], [تاریکی...   \n",
       "\n",
       "                                                       idx_word_nmv  \\\n",
       "book    idx_chapter idx_verse                                         \n",
       "Genesis 0           0                               [1, 3, 4, 6, 8]   \n",
       "                    0                               [1, 3, 4, 6, 8]   \n",
       "                    0                               [1, 3, 4, 6, 8]   \n",
       "                    0                               [1, 3, 4, 6, 8]   \n",
       "                    0                               [1, 3, 4, 6, 8]   \n",
       "                    0                               [1, 3, 4, 6, 8]   \n",
       "                    1          [0, 1, 3, 7, 10, 13, 14, 15, 16, 18]   \n",
       "                    1          [0, 1, 3, 7, 10, 13, 14, 15, 16, 18]   \n",
       "                    1          [0, 1, 3, 7, 10, 13, 14, 15, 16, 18]   \n",
       "                    1          [0, 1, 3, 7, 10, 13, 14, 15, 16, 18]   \n",
       "                    1          [0, 1, 3, 7, 10, 13, 14, 15, 16, 18]   \n",
       "                    1          [0, 1, 3, 7, 10, 13, 14, 15, 16, 18]   \n",
       "                    1          [0, 1, 3, 7, 10, 13, 14, 15, 16, 18]   \n",
       "                    1          [0, 1, 3, 7, 10, 13, 14, 15, 16, 18]   \n",
       "                    1          [0, 1, 3, 7, 10, 13, 14, 15, 16, 18]   \n",
       "                    1          [0, 1, 3, 7, 10, 13, 14, 15, 16, 18]   \n",
       "                    1          [0, 1, 3, 7, 10, 13, 14, 15, 16, 18]   \n",
       "\n",
       "                              idx_wordmatch_nmv wordmatch_nmv pos_tag_nmv  \\\n",
       "book    idx_chapter idx_verse                                               \n",
       "Genesis 0           0                         3           خدا    [خدا, N]   \n",
       "                    0                         3           خدا    [خدا, N]   \n",
       "                    0                         3           خدا    [خدا, N]   \n",
       "                    0                         3           خدا    [خدا, N]   \n",
       "                    0                         3           خدا    [خدا, N]   \n",
       "                    0                         3           خدا    [خدا, N]   \n",
       "                    1                        14           خدا    [خدا, N]   \n",
       "                    1                        14           خدا    [خدا, N]   \n",
       "                    1                        14           خدا    [خدا, N]   \n",
       "                    1                        14           خدا    [خدا, N]   \n",
       "                    1                        14           خدا    [خدا, N]   \n",
       "                    1                        14           خدا    [خدا, N]   \n",
       "                    1                         0          زمین  [زمین, Ne]   \n",
       "                    1                        14           خدا    [خدا, N]   \n",
       "                    1                        14           خدا    [خدا, N]   \n",
       "                    1                        14           خدا    [خدا, N]   \n",
       "                    1                        14           خدا    [خدا, N]   \n",
       "\n",
       "                                    cosine_similarity  \n",
       "book    idx_chapter idx_verse                          \n",
       "Genesis 0           0          [[0.5750017952637303]]  \n",
       "                    0          [[0.5802883873468905]]  \n",
       "                    0          [[0.5861434809310097]]  \n",
       "                    0          [[0.6244082085960594]]  \n",
       "                    0          [[0.6315576137669605]]  \n",
       "                    0          [[0.6730095257915146]]  \n",
       "                    1          [[0.5539344012703646]]  \n",
       "                    1           [[0.569367715437644]]  \n",
       "                    1          [[0.5956766805237825]]  \n",
       "                    1          [[0.6016001426782509]]  \n",
       "                    1          [[0.6606292771967468]]  \n",
       "                    1          [[0.6650169715287683]]  \n",
       "                    1          [[0.6706229051016932]]  \n",
       "                    1          [[0.6730095636119798]]  \n",
       "                    1          [[0.6836679391320224]]  \n",
       "                    1          [[0.7052896188538254]]  \n",
       "                    1          [[0.7052896188538254]]  "
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df.explode(\n",
    "    column=[\n",
    "        \"idx_word_original\",\n",
    "        \"eng_word\",\n",
    "        \"strongs\",\n",
    "        \"morphology\",\n",
    "        \"idx_wordmatch_nmv\",\n",
    "        \"wordmatch_nmv\",\n",
    "        \"pos_tag_nmv\",\n",
    "        \"cosine_similarity\"\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Worked really well! except it used אלהים twice - for خدا and آفرید. Find a way to limit to one use per word.\n",
    "Has a problem with names and confusing Jesus with Christ. Check titles of God.\n",
    "\n",
    "Low similarity matches for prepositions etc. Consider removing stopwords to avoid incorrect matches.\n",
    "\n",
    "Resolve by Part of Speech (POS) tagging and looking for matches in morphological codes that match the POS tag.\n",
    "Also option to use Named Entity Recognition (NER) to match proper nouns. Could transliterate instead of translating and match that way if names are still a problem.\n",
    "Need to find a tokenizer that deals with farsi multi-word expressions e.g. verbs that are a couplet of noun and verb like Doha mikonam - prayer I do.\n",
    "SpaCy library farsi language resources might help do this to prepare the dataframe for feeding into BERT in this notebook.\n",
    "\n",
    "Resolve matching multiple times by finding best farsi match to original word instead of other way round. Removing stopwords from original before matching will also help the above issue.\n",
    "Use idx_word to match the resulting match back to the farsi text in the right order and allow for added farsi words without a match in the original."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = [1,2,3,4,5,6]\n",
    "b = [7,8,9,10,11,12]\n",
    "\n",
    "c = [str(ai)+\"-\"+str(bi) for ai in a for bi in b]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['1-7',\n",
       " '1-8',\n",
       " '1-9',\n",
       " '1-10',\n",
       " '1-11',\n",
       " '1-12',\n",
       " '2-7',\n",
       " '2-8',\n",
       " '2-9',\n",
       " '2-10',\n",
       " '2-11',\n",
       " '2-12',\n",
       " '3-7',\n",
       " '3-8',\n",
       " '3-9',\n",
       " '3-10',\n",
       " '3-11',\n",
       " '3-12',\n",
       " '4-7',\n",
       " '4-8',\n",
       " '4-9',\n",
       " '4-10',\n",
       " '4-11',\n",
       " '4-12',\n",
       " '5-7',\n",
       " '5-8',\n",
       " '5-9',\n",
       " '5-10',\n",
       " '5-11',\n",
       " '5-12',\n",
       " '6-7',\n",
       " '6-8',\n",
       " '6-9',\n",
       " '6-10',\n",
       " '6-11',\n",
       " '6-12']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.10 ('env': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "99d7c05818e589d4400664f97bf9869404f8254e09c4db3a9c44ab121cda7655"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}