2_FromPlural2Singular.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7fa062f6",
   "metadata": {},
   "source": [
    "Singular and plural instances of the same word counts as separate words. \n",
    "\n",
    "The lemma of each Doc is able to recognice the singular word only when we use the large corpus:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "40c0f17c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pysrt\n",
    "import spacy\n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "from copy import deepcopy\n",
    "from tabulate import tabulate"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "187842cc",
   "metadata": {},
   "source": [
    "Example:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "bad698c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "donor\n",
      "NNS\n",
      "donors\n",
      "NOUN\n",
      "...\n",
      "nanny\n",
      "NNS\n",
      "nannies\n",
      "NOUN\n",
      "...\n",
      "child\n",
      "NNS\n",
      "children\n",
      "NOUN\n",
      "...\n",
      "child\n",
      "NN\n",
      "child\n",
      "NOUN\n",
      "...\n"
     ]
    }
   ],
   "source": [
    "nlp = spacy.load(\"en_core_web_lg\")\n",
    "doc = nlp(\"donors nannies children child\")\n",
    "\n",
    "for t in doc:\n",
    "    print(t.lemma_)\n",
    "    print(t.tag_)\n",
    "    print(t.text)\n",
    "    print(t.pos_)\n",
    "    print('...')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "cbed6416",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "00:01:19,727 --> 00:01:23,356\n",
      "white storks wanted pick bird\n",
      "\n",
      "2\n",
      "00:01:23,567 --> 00:01:26,400\n",
      "representative birds world\n",
      "\n",
      "3\n",
      "00:01:26,447 --> 00:01:29,200\n",
      "worse pick white stork\n",
      "\n",
      "4\n",
      "00:01:29,327 --> 00:01:32,637\n",
      "marvellous flyer intrepid traveller\n",
      "\n",
      "5\n",
      "\n"
     ]
    }
   ],
   "source": [
    "f = open(\"preprocessed_sub.txt\")\n",
    "doc = nlp(f.read())\n",
    "\n",
    "print(doc[:50]) # first 500 characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "634c37c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plural2sing(doc, pos_filter={}):\n",
    "    \"\"\"\n",
    "    Splits documents into tokens, change plural tokens to singular and lemmatizes the\n",
    "    text.\n",
    "    \n",
    "    Parameters:\n",
    "    doc (spacy.document)\n",
    "    \"\"\"\n",
    "    out = list()\n",
    "    for subtitle in doc:\n",
    "        s = []\n",
    "        for token in subtitle:\n",
    "            if token.tag_ == \"NNS\":\n",
    "                s.append(token.lemma_)\n",
    "            else:\n",
    "                s.append(token.text.lower())\n",
    "        out.append(s)\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "dcbe543e",
   "metadata": {},
   "outputs": [],
   "source": [
    "utterances = [subtitle.text for subtitle in subtitles]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "703757ee",
   "metadata": {},
   "source": [
    "Here we can see how Counter recognizes \"bird\" and \"birds\" as separate words:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "14ea8f02",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['birds' 'bird' 'feathers' 'like' 'wings' 'air' 'great' 'way' 'display'\n",
      " 'flight' 'nest' 'beak' 'africa' 'body' 'sun']\n"
     ]
    }
   ],
   "source": [
    "text = ' '.join(utterances)\n",
    "text_for_counts = text.split(\" \")\n",
    "\n",
    "counts = Counter(text_for_counts)\n",
    "labels, values = zip(*counts.items())\n",
    "\n",
    "# sort your values in descending order\n",
    "indSort = np.argsort(values)[::-1]\n",
    "\n",
    "# rearrange your data\n",
    "labels = np.array(labels)[indSort]\n",
    "values = np.array(values)[indSort]\n",
    "print(labels[:15])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0fd780db",
   "metadata": {},
   "source": [
    "Change plural words by singular words:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "e34bb6f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['white', 'stork', 'wanted', 'pick', 'bird'],\n",
       " ['representative', 'bird', 'world'],\n",
       " ['worse', 'pick', 'white', 'stork'],\n",
       " ['marvellous', 'flyer', 'intrepid', 'traveller'],\n",
       " ['pair', 'come', 'africa', 'nest', 'small', 'town', 'bavaria'],\n",
       " ['complicated', 'courtship', 'greeting', 'ritual'],\n",
       " ['devoted', 'parent'],\n",
       " ['stand', 'bird', 'world', 'stork', 'feather'],\n",
       " ['seen', 'key', 'crucial', 'bird'],\n",
       " ['feather', 'marvellous', 'aerofoil'],\n",
       " ['man', 'invent', 'strong', 'weight', 'weight'],\n",
       " ['extremely', 'efficient', 'insulator', 'important', 'bird'],\n",
       " ['complicated', 'structure'],\n",
       " ['feather', 'separate', 'filament'],\n",
       " ['central', 'quill']]"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus = nlp.pipe(utterances, batch_size=26000)\n",
    "\n",
    "processed = plural2sing(corpus)\n",
    "\n",
    "processed[:15]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "d5468fdb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def flatten(xss):\n",
    "    return [x for xs in xss for x in xs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "60524fc7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['bird' 'feather' 'wing' 'like' 'nest' 'beak' 'air' 'stork' 'display'\n",
      " 'great' 'way' 'egg' 'flight' 'body' 'young']\n"
     ]
    }
   ],
   "source": [
    "flat_processed = flatten(processed)\n",
    "text = ' '.join(flat_processed)\n",
    "text_for_counts = text.split(\" \")\n",
    "\n",
    "counts = Counter(text_for_counts)\n",
    "labels, values = zip(*counts.items())\n",
    "\n",
    "# sort your values in descending order\n",
    "indSort = np.argsort(values)[::-1]\n",
    "\n",
    "# rearrange your data\n",
    "labels = np.array(labels)[indSort]\n",
    "values = np.array(values)[indSort]\n",
    "print(labels[:15])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "f6133aa3",
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessed_subtitles = deepcopy(subtitles)\n",
    "\n",
    "for subtitle, proc_subtitle in zip(preprocessed_subtitles, processed):\n",
    "    subtitle.text = \" \".join(proc_subtitle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "dfe72721",
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessed_subtitles.save('./preprocessed_singular_sub.txt')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}