vocabulary.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Compute vocabulary\n",
    "Compute two sets of vocabulary\n",
    "- **Mixed set**: vocabulary where all words may not be correctly spelled\n",
    "- **Clean set**: vocabulary where all words are correctly spelled"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "from collections import Counter\n",
    "from typing import List\n",
    "\n",
    "from indicnlp.tokenize.indic_tokenize import trivial_tokenize_indic\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Utilities"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "def tokenize_text(text: List[str]) -> List[List[str]]:\n",
    "    \"\"\"Tokenize text\"\"\"\n",
    "    return [trivial_tokenize_indic(sent) for sent in tqdm(text, desc='tokenize', unit=' sentences')]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "source": [
    "def build_vocab(tokenized_text: List[List[str]]) -> Counter:\n",
    "    \"\"\"Build vocabulary from tokenized text\"\"\"\n",
    "    counter = Counter()\n",
    "    for toks in tqdm(tokenized_text, desc='build vocab and compute frequencies of tokens', unit=' sentences'):\n",
    "        counter.update(toks)\n",
    "    print(f'Number of sentences: {len(tokenized_text):,}')\n",
    "    print(f'Number of unique words or equivalently, the size of vocabulary: {len(counter):,}')\n",
    "    print(f'Number of running words: {sum([freq for _, freq in counter.items()]):,}')\n",
    "    return counter"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": 3,
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "def write_vocab_to_json(counter: Counter, json_path: str) -> None:\n",
    "    \"\"\"Write vocabulary to json file\"\"\"\n",
    "    json_str = json.dumps(dict(counter), ensure_ascii=False, indent=4, )\n",
    "    with open(json_path, 'w', encoding='utf-8') as f_json:\n",
    "        f_json.write(json_str)\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Mixed vocabulary"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Load mixed data and tokenize\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "read lines from file: 100%|██████████| 3594672/3594672 [00:01<00:00, 1840526.21it/s]\n"
     ]
    }
   ],
   "source": [
    "mixed_data_filepath = os.path.join('data/or')\n",
    "assert os.path.isfile(mixed_data_filepath)  # sanity check\n",
    "with open(mixed_data_filepath, 'r', encoding='utf-8') as f:\n",
    "    lines = [s.strip() for s in tqdm(f.readlines(), desc='read lines from file')]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "tokenize: 100%|██████████| 3594672/3594672 [01:31<00:00, 39420.61 sentences/s]\n"
     ]
    }
   ],
   "source": [
    "# tokenize\n",
    "mixed_tokens = tokenize_text(lines)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Build mixed vocabulary\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "build vocab and compute frequencies of tokens: 100%|██████████| 3594672/3594672 [00:24<00:00, 147012.36 sentences/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of sentences: 3,594,672\n",
      "Number of unique words or equivalently, the size of vocabulary: 778,862\n",
      "Number of running words: 51,151,273\n"
     ]
    }
   ],
   "source": [
    "mixed_vocab_counter = build_vocab(mixed_tokens)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "data": {
      "text/plain": "[('।', 3393061),\n (',', 1191253),\n ('ଓ', 534792),\n ('ଏହି', 437185),\n ('ପାଇଁ', 373726),\n ('ସେ', 240775),\n ('ବୋଲି', 239837),\n ('ପରେ', 224959),\n ('କରି', 221628),\n ('ଏକ', 213516),\n ('ମଧ୍ୟ', 210907),\n ('ଏବଂ', 198988),\n ('କରିଥିଲେ', 195168),\n ('ସହ', 177040),\n ('-', 174796),\n ('ଖବର', 169373),\n ('.', 166728),\n ('କରିବା', 166276),\n ('ନେଇ', 161728),\n ('ବେଳେ', 156327)]"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# most common words\n",
    "mixed_vocab_counter.most_common(n=20)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [],
   "source": [
    "# write to json file\n",
    "write_vocab_to_json(mixed_vocab_counter, os.path.join('mixed_vocab_counter.json'))\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Clean vocabulary"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Load clean data and tokenize"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "source": [
    "clean_data_filepath = os.path.join('data/Odia_structured_wordlist.json')\n",
    "assert os.path.isfile(clean_data_filepath)  # sanity check\n",
    "with open(clean_data_filepath, 'r', encoding='utf-8') as f:\n",
    "    di = json.load(f)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": 11,
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [
    {
     "data": {
      "text/plain": "{'gender': None,\n 'pronunciation': 'Haḻka pāibā',\n 'raw': ' (ଅମୁକର) ହଳକ ପାଇବା— Haḻka pāibā [synonym(s): হলক পাত্তয়া हलक पाना] ଦେ. ବି— 1। [କୌଣସି କାର୍ଯ୍ୟ କରିବାପାଇଁ] ସାହସ ଉପୁଜିବା। 1. Having the boldness or courage (to undertake a particular work). (ଯଥା—ଏଡ଼େ ଡେଙ୍ଗା ଗଛରେ ଚଢ଼ିବାକୁ ମୋର ହଲକ୍ ପାଉ ନାହିଁ।) ',\n 'synonyms': '[synonym(s): হলক পাত্তয়া हलक पाना]',\n 'word_details': [{'juktakhyara': 'ଦେ. ବି',\n   'meaning': '1। [କୌଣସି କାର୍ଯ୍ୟ କରିବାପାଇଁ] ସାହସ ଉପୁଜିବା। 1. Having the boldness or courage (to undertake a particular work). (ଯଥା—ଏଡ଼େ ଡେଙ୍ଗା ଗଛରେ ଚଢ଼ିବାକୁ ମୋର ହଲକ୍ ପାଉ ନାହିଁ।)',\n   'verse': None}]}"
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "di[list(di.keys())[4]]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "tokenize: 100%|██████████| 121658/121658 [00:00<00:00, 148792.93 sentences/s]\n"
     ]
    }
   ],
   "source": [
    "# collect all lines first\n",
    "lines = []\n",
    "for k, v in di.items():\n",
    "    lines.append(k)\n",
    "    if isinstance(v, str):\n",
    "        lines.append(v)\n",
    "    elif isinstance(v, list):\n",
    "        for d in v:\n",
    "            for _k, _v in d.items():\n",
    "                lines.append(_k)\n",
    "                lines.append(_v)\n",
    "    else:\n",
    "        pass\n",
    "\n",
    "# remove None\n",
    "lines = list(filter(lambda x: x is not None, lines))\n",
    "\n",
    "# tokenize\n",
    "clean_tokens = tokenize_text(lines)\n",
    "\n",
    "# remove non Odia words\n",
    "english_letters = [c for c in 'abcdefghijklmnopqrstuvwxyz' + 'abcdefghijklmnopqrstuvwxyz'.upper()]\n",
    "odia_clean_tokens = []\n",
    "for tokens in clean_tokens:\n",
    "    odia_clean_tokens.append([token for token in tokens if len(set(token) & set(english_letters)) == 0])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "source": [
    "odia_clean_tokens[1000:1020]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": 14,
   "outputs": [
    {
     "data": {
      "text/plain": "[['ଅଗଲ୍କଟା'],\n ['ଅଗଳା'],\n ['ଅଗଷ୍ଟ'],\n ['ଅଗସ୍ତ'],\n ['ଅଗସ୍ତି'],\n ['ଅଗସ୍ତ୍ୟ'],\n ['ଅଗସ୍ତ୍ୟ', 'ଯାତ୍ରା'],\n ['ଅଗସ୍ତ୍ୟ', 'ସଂହିତା'],\n ['ଅଗସ୍ତ୍ୟକୂଟ'],\n ['ଅଗସ୍ତ୍ୟସର'],\n ['ଅଗସ୍ତ୍ୟାଶ୍ରମ'],\n ['ଅଗସ୍ତ୍ୟୋଦୟ'],\n ['ଅଗା'],\n ['ଅଗାଣ୍ଡିଆ'],\n ['ଅଗାତ୍ମଜା'],\n ['ଅଗାଦ'],\n ['ଅଗାଧ'],\n ['ଅଗାଧୁ'],\n ['ଅଗାଧୁଆ'],\n ['ଅଗାଧୁଆ', 'ଅପାଧୁଆ']]"
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Build clean vocabulary"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "build vocab and compute frequencies of tokens: 100%|██████████| 121658/121658 [00:00<00:00, 230931.92 sentences/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of sentences: 121,658\n",
      "Number of unique words or equivalently, the size of vocabulary: 93,378\n",
      "Number of running words: 170,497\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "clean_vocab_counter = build_vocab(odia_clean_tokens)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [
    {
     "data": {
      "text/plain": "[(')', 2750),\n ('(', 2739),\n ('କରିବା', 1623),\n ('ଇତ୍ୟାଦି', 1472),\n ('ଧାତୁ', 1147),\n ('ହେବା', 911),\n ('ମାରିବା', 704),\n ('ଦେବା', 567),\n ('ହାତ', 268),\n ('ପଡ଼ିବା', 253),\n ('ରକ୍ତ', 244),\n ('ପାଣି', 227),\n ('ମନ', 208),\n ('ମୁହଁ', 203),\n ('ଯିବା', 202),\n ('ବେ', 194),\n ('ରଖିବା', 177),\n ('ପକାଇବା', 176),\n ('ଧରିବା', 171),\n ('ରସ', 161)]"
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# most common words\n",
    "clean_vocab_counter.most_common(n=20)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [],
   "source": [
    "# write to json file\n",
    "write_vocab_to_json(clean_vocab_counter, os.path.join('clean_vocab_counter.json'))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}