wiki_bigrams_by_category.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python version + compiler & build info: 3.6.5 |Anaconda custom (64-bit)| (default, Mar 29 2018, 18:21:58) \n",
      "[GCC 7.2.0]\n",
      "Jupyter notebook version: 5.4.1\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from ipywidgets import FloatProgress, IntText\n",
    "import time\n",
    "import collections\n",
    "import re\n",
    "import pickle\n",
    "import sys, os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from math import log\n",
    "\n",
    "print(\"Python version + compiler & build info:\", sys.version)\n",
    "print(\"Jupyter notebook version:\", os.popen(\"jupyter notebook --version\").readlines()[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/is_male.pkl', 'rb') as infile:\n",
    "    is_male = pickle.load(infile)\n",
    "\n",
    "def save_object(obj, filename):\n",
    "    with open(filename, 'wb') as outfile:  \n",
    "        pickle.dump(obj, outfile, pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting bigrams from Wiki biographies, according to gender and category\n",
    "Categories: artists, monarchs, programming, scientist, sports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_bigrams(text):\n",
    "    return zip(*[text[i:] for i in range(2)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c6d4faabbe72430486931ded36e7d01b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "IntText(value=0, description='Categories')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "943941cba349495081dd397738c1ce39",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, max=850399.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fp = FloatProgress(min=0, max=850399)\n",
    "cc = IntText(value=0, description=\"Categories\")\n",
    "display(cc, fp)\n",
    "article_count = 0\n",
    "category_counts = None\n",
    "\n",
    "matcher = re.compile(\"\\[\\[Category:([^\\]]*)\", re.IGNORECASE)\n",
    "\n",
    "category_to_articles = collections.defaultdict(list)\n",
    "\n",
    "with open(\"data/gendered-labeled-articles\", 'r') as f:\n",
    "    for line in f:\n",
    "        article_count += 1\n",
    "        if article_count % 1000 == 0:\n",
    "            fp.value = article_count\n",
    "            cc.value = len(category_to_articles)\n",
    "        article_id = int(line.split(\" \", 1)[0])\n",
    "        for category in matcher.findall(line):\n",
    "            category_to_articles[category].append(article_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'1809 births',\n",
       " '1865 deaths',\n",
       " '19th-century American politicians',\n",
       " '19th-century Christians',\n",
       " 'Abraham Lincoln| ',\n",
       " 'American Christians',\n",
       " 'American people of English descent',\n",
       " 'American postmasters',\n",
       " 'American surveyors',\n",
       " 'Assassinated Presidents of the United States',\n",
       " 'Assassinated heads of state',\n",
       " 'Burials at Oak Ridge Cemetery',\n",
       " 'Deaths by firearm in Washington, D.C.',\n",
       " 'Hall of Fame for Great Americans inductees',\n",
       " 'Illinois Central Railroad people'}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "{k for k in list(category_to_articles)[:15]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('sports', 22673),\n",
       " ('monarch', 12644),\n",
       " ('scientist', 10384),\n",
       " ('artists', 30265),\n",
       " ('programming', 1047)]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_to_articles = collections.defaultdict(set)\n",
    "topic_patterns = {\n",
    "    \"programming\": re.compile(\"software|programmer\"),\n",
    "    \"sports\": re.compile(\"athlete|sportspeople\"),\n",
    "    \"artists\": re.compile(\"artist\"),\n",
    "    \"scientist\": re.compile(\"scientist\"),\n",
    "#     \"fictional\": re.compile(\"fictional\"),\n",
    "    \"monarch\": re.compile(\"king|queen|monarch|royal|prince|princess\"),\n",
    "}\n",
    "\n",
    "for k,v in category_to_articles.items():\n",
    "    for topic, pattern in topic_patterns.items():\n",
    "        if pattern.search(k):\n",
    "            topic_to_articles[topic].update(v)\n",
    "            \n",
    "[(k, len(v)) for (k,v) in topic_to_articles.items()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "214"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Note there is some overlap between categories\n",
    "len(topic_to_articles['monarch'].intersection(topic_to_articles['sports']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "75953"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create set of all article ids from all 5 categories\n",
    "all_ids = set.union(*[v for k,v in topic_to_articles.items()])\n",
    "len(all_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c700665c2d6b4998be837efe103c452b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, max=850399.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a1bde5810cc4e4ebb527e2ea6930650",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "IntText(value=0, description='Article Count:')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Articles: 850399  Time: 1042.0519289970398 seconds\n"
     ]
    }
   ],
   "source": [
    "master_dict = {\n",
    "    'male_sports': collections.Counter(), 'female_sports': collections.Counter(),\n",
    "    'male_monarch': collections.Counter(), 'female_monarch': collections.Counter(),\n",
    "    'male_artists': collections.Counter(), 'female_artists': collections.Counter(),\n",
    "    'male_scientist': collections.Counter(), \n",
    "    'female_scientist': collections.Counter(),\n",
    "    'male_programming': collections.Counter(), \n",
    "    'female_programming': collections.Counter() }\n",
    "cats = ['sports', 'monarch', 'artists', 'scientist', 'programming']\n",
    "cat_count = {'male_sports': 0, 'female_sports': 0, 'male_monarch': 0, \n",
    "             'female_monarch': 0, 'male_artists': 0, 'female_artists': 0, \n",
    "             'male_scientist': 0, 'female_scientist': 0, 'male_programming': 0, \n",
    "             'female_programming': 0}\n",
    "\n",
    "fp = FloatProgress(min=0, max=850399)\n",
    "all_articles = IntText(value=0, description=\"Article Count:\")\n",
    "display(fp, all_articles)\n",
    "article_count = 0\n",
    "start = time.time()\n",
    "try:\n",
    "    with open(\"data/gendered-labeled-articles.stripped\", 'r') as f:\n",
    "        for line in f:\n",
    "            article_count += 1\n",
    "            all_articles.value += 1\n",
    "            if article_count % 1000 == 0:\n",
    "                fp.value = article_count\n",
    "            line = line.split()\n",
    "            art_id = line.pop(0)\n",
    "            if int(art_id) in all_ids:\n",
    "                male = is_male[art_id]\n",
    "                line = [line[i-1 : i+99] for i in range(1, len(line), 99)]\n",
    "                if male:\n",
    "                    for c in cats:\n",
    "                        if int(art_id) in topic_to_articles[c]:\n",
    "                            cat_count[\"male_\" + c] += 1\n",
    "                else:\n",
    "                    for c in cats:\n",
    "                        if int(art_id) in topic_to_articles[c]:\n",
    "                            cat_count[\"female_\" + c] += 1\n",
    "                while line:\n",
    "                    bigrams = list(get_bigrams(line.pop(0)))\n",
    "                    for c in cats:\n",
    "                        if int(art_id) in topic_to_articles[c]:\n",
    "                            if male:\n",
    "                                master_dict[\"male_\" + c].update(bigrams)\n",
    "                            else:\n",
    "                                master_dict[\"female_\" + c].update(bigrams)\n",
    "            # save male bigram counters in batches to save memory\n",
    "            if article_count % 50000 == 0:\n",
    "                for c in cats[:4]:\n",
    "                    save_object(master_dict[\"male_\" + c], \n",
    "                                'data/bigrams/male_{}_{}.pkl'.format(c, article_count//50000))\n",
    "                    master_dict[\"male_\" + c] = collections.Counter()\n",
    "    # last iteration:\n",
    "    for c in cats[:4]:\n",
    "        save_object(master_dict[\"male_\" + c], \n",
    "                    'data/bigrams/male_{}_{}.pkl'.format(c, article_count//50000 + 1))\n",
    "        master_dict[\"male_\" + c] = collections.Counter()\n",
    "finally:\n",
    "    print(\"Articles:\", article_count, \" Time:\", (time.time() - start), \"seconds\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save all\n",
    "save_object(master_dict[\"male_programming\"], 'data/bigrams/male_programming.pkl')\n",
    "\n",
    "for c in cats:\n",
    "    save_object(master_dict[\"female_\" + c], 'data/bigrams/female_{}.pkl'.format(c))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('male_sports', 17528),\n",
       " ('female_sports', 5145),\n",
       " ('male_monarch', 10120),\n",
       " ('female_monarch', 2524),\n",
       " ('male_artists', 21781),\n",
       " ('female_artists', 8484),\n",
       " ('male_scientist', 8438),\n",
       " ('female_scientist', 1946),\n",
       " ('male_programming', 995),\n",
       " ('female_programming', 52)]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(k, v) for (k,v) in cat_count.items()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('male_sports', 0),\n",
       " ('female_sports', 982573),\n",
       " ('male_monarch', 0),\n",
       " ('female_monarch', 1299352),\n",
       " ('male_artists', 0),\n",
       " ('female_artists', 3854438),\n",
       " ('male_scientist', 0),\n",
       " ('female_scientist', 956458),\n",
       " ('male_programming', 551059),\n",
       " ('female_programming', 54269)]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# bigram counters still in kernel\n",
    "[(k, len(v)) for (k,v) in master_dict.items()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6f1ef7da1c0e494cbc5263576d42b97c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, max=18.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4079579ccca04942967a3b56d4eb2d8d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "IntText(value=0, description='Count:')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time: 30.27496027946472 seconds\n"
     ]
    }
   ],
   "source": [
    "# Create master counters of bigrams for male categories \n",
    "# of sports, monarch, scientist, artists:\n",
    "\n",
    "# Sports\n",
    "fp = FloatProgress(min=0, max=18)\n",
    "count = IntText(value=0, description=\"Count:\")\n",
    "display(fp, count)\n",
    "male_sports = collections.Counter()\n",
    "start = time.time()\n",
    "for i in range(1, 19):\n",
    "    fp.value += 1\n",
    "    count.value += 1\n",
    "    with open('data/bigrams/male_sports_{}.pkl'.format(i), 'rb') as infile:\n",
    "        male_sports += pickle.load(infile)\n",
    "for k in set(k for k,v in male_sports.items() if v <= 2):\n",
    "    del male_sports[k]  \n",
    "save_object(male_sports, 'data/bigrams/male_sports.pkl')\n",
    "print(\"Time:\", (time.time() - start), \"seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deleting temporary batches...\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "print(\"Deleting temporary batches...\")\n",
    "for i in range(1, 19):\n",
    "    os.remove('data/bigrams/male_sports_{}.pkl'.format(i))\n",
    "print(\"Done.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1e0700938f149808ef10ecd6e9045ba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, max=18.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c547e89189f4ac1b70fabaa6c7900c9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "IntText(value=0, description='Count:')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time: 34.27636647224426 seconds\n"
     ]
    }
   ],
   "source": [
    "# Monarch\n",
    "fp = FloatProgress(min=0, max=18)\n",
    "count = IntText(value=0, description=\"Count:\")\n",
    "display(fp, count)\n",
    "male_monarch = collections.Counter()\n",
    "start = time.time()\n",
    "for i in range(1, 19):\n",
    "    fp.value += 1\n",
    "    count.value += 1\n",
    "    with open('data/bigrams/male_monarch_{}.pkl'.format(i), 'rb') as infile:\n",
    "        male_monarch += pickle.load(infile)\n",
    "for k in set(k for k,v in male_monarch.items() if v <= 2):\n",
    "    del male_monarch[k]  \n",
    "save_object(male_monarch, 'data/bigrams/male_monarch.pkl')\n",
    "print(\"Time:\", (time.time() - start), \"seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deleting temporary batches...\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "print(\"Deleting temporary batches...\")\n",
    "for i in range(1, 19):\n",
    "    os.remove('data/bigrams/male_monarch_{}.pkl'.format(i))\n",
    "print(\"Done.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "06d19fe15112454eaf9009a1969f1756",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, max=18.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bf24d58e54ad4c8bbf8674c72b14e983",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "IntText(value=0, description='Count:')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time: 26.785165071487427 seconds\n"
     ]
    }
   ],
   "source": [
    "# Scientist\n",
    "fp = FloatProgress(min=0, max=18)\n",
    "count = IntText(value=0, description=\"Count:\")\n",
    "display(fp, count)\n",
    "male_scientist = collections.Counter()\n",
    "start = time.time()\n",
    "for i in range(1, 19):\n",
    "    fp.value += 1\n",
    "    count.value += 1\n",
    "    with open('data/bigrams/male_scientist_{}.pkl'.format(i), 'rb') as infile:\n",
    "        male_scientist += pickle.load(infile)\n",
    "for k in set(k for k,v in male_scientist.items() if v <= 2):\n",
    "    del male_scientist[k]  \n",
    "save_object(male_scientist, 'data/bigrams/male_scientist.pkl')\n",
    "print(\"Time:\", (time.time() - start), \"seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deleting temporary batches...\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "print(\"Deleting temporary batches...\")\n",
    "for i in range(1, 19):\n",
    "    os.remove('data/bigrams/male_scientist_{}.pkl'.format(i))\n",
    "print(\"Done.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dc94d1802e0a44ed979015e63a8673a0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, max=18.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7099c660d6ba4df3a0db06c159482192",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "IntText(value=0, description='Count:')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time: 52.9793004989624 seconds\n"
     ]
    }
   ],
   "source": [
    "# Artist\n",
    "fp = FloatProgress(min=0, max=18)\n",
    "count = IntText(value=0, description=\"Count:\")\n",
    "display(fp, count)\n",
    "male_artists = collections.Counter()\n",
    "start = time.time()\n",
    "for i in range(1, 19):\n",
    "    fp.value += 1\n",
    "    count.value += 1\n",
    "    with open('data/bigrams/male_artists_{}.pkl'.format(i), 'rb') as infile:\n",
    "        male_artists += pickle.load(infile)\n",
    "for k in set(k for k,v in male_artists.items() if v <= 3):\n",
    "    del male_artists[k]  \n",
    "save_object(male_artists, 'data/bigrams/male_artists.pkl')\n",
    "print(\"Time:\", (time.time() - start), \"seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deleting temporary batches...\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "print(\"Deleting temporary batches...\")\n",
    "for i in range(1, 19):\n",
    "    os.remove('data/bigrams/male_artists_{}.pkl'.format(i))\n",
    "print(\"Done.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of bigrams in men's bios, sports category: 21443336\n",
      "Number of bigrams in men's bios, monarch category: 14567320\n",
      "Number of bigrams in men's bios, scientist category: 9049407\n",
      "Number of bigrams in men's bios, artists category: 29899078\n"
     ]
    }
   ],
   "source": [
    "# The missing values from above\n",
    "print(\"Number of bigrams in men's bios, sports category:\", sum(male_sports.values()))\n",
    "print(\"Number of bigrams in men's bios, monarch category:\", sum(male_monarch.values()))\n",
    "print(\"Number of bigrams in men's bios, scientist category:\", sum(male_scientist.values()))\n",
    "print(\"Number of bigrams in men's bios, artists category:\", sum(male_artists.values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To load the data back in:\n",
    "\n",
    "with open('data/bigrams/female_sports.pkl', 'rb') as infile:\n",
    "    female_sports = pickle.load(infile)\n",
    "with open('data/bigrams/female_monarch.pkl', 'rb') as infile:\n",
    "    female_monarch = pickle.load(infile)\n",
    "with open('data/bigrams/female_scientist.pkl', 'rb') as infile:\n",
    "    female_scientist = pickle.load(infile)\n",
    "with open('data/bigrams/female_artists.pkl', 'rb') as infile:\n",
    "    female_artists = pickle.load(infile)\n",
    "with open('data/bigrams/female_programming.pkl', 'rb') as infile:\n",
    "    female_programming = pickle.load(infile)\n",
    "with open('data/bigrams/male_programming.pkl', 'rb') as infile:\n",
    "    male_programming = pickle.load(infile)\n",
    "with open('data/bigrams/male_scientist.pkl', 'rb') as infile:\n",
    "    male_scientist = pickle.load(infile)\n",
    "with open('data/bigrams/male_sports.pkl', 'rb') as infile:\n",
    "    male_sports = pickle.load(infile)\n",
    "with open('data/bigrams/male_artists.pkl', 'rb') as infile:\n",
    "    male_artists = pickle.load(infile)\n",
    "with open('data/bigrams/male_monarch.pkl', 'rb') as infile:\n",
    "    male_monarch = pickle.load(infile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Category</th>\n",
       "      <th>All bios</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sports</td>\n",
       "      <td>22673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>monarch</td>\n",
       "      <td>12644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>scientist</td>\n",
       "      <td>10384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>artists</td>\n",
       "      <td>30265</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>programming</td>\n",
       "      <td>1047</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Category  All bios\n",
       "0       sports     22673\n",
       "1      monarch     12644\n",
       "2    scientist     10384\n",
       "3      artists     30265\n",
       "4  programming      1047"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cols = [(k, len(v)) for (k,v) in topic_to_articles.items()]\n",
    "# cols.insert(0, (\"All categories\", 848989))\n",
    "\n",
    "df = pd.DataFrame(cols, columns=[\"Category\", \"All bios\"])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('male_sports', 17528),\n",
       " ('male_monarch', 10120),\n",
       " ('male_artists', 21781),\n",
       " ('male_scientist', 8438),\n",
       " ('male_programming', 995)]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Men\n",
    "men_cats = [(k,v) for (k,v) in cat_count.items()][0::2]\n",
    "# men_cats.insert(0, ('all biographies', 718413))\n",
    "men_cats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('female_sports', 5145),\n",
       " ('female_monarch', 2524),\n",
       " ('female_artists', 8484),\n",
       " ('female_scientist', 1946),\n",
       " ('female_programming', 52)]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Women\n",
    "women_cats = [(k, v) for (k,v) in cat_count.items()][1::2]\n",
    "# women_cats.insert(0, ('all biographies', 130576))\n",
    "women_cats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ensuring correct order\n",
    "df[\"Men\"] = [men_cats[i][1] for i in [0, 1, 3, 2, 4]]\n",
    "df[\"Women\"] = [women_cats[i][1] for i in [0, 1, 3, 2, 4]]\n",
    "df[\"% Women\"] = round(df[\"Women\"] / df[\"All bios\"] * 100,1)\n",
    "df.sort_values(by=\"% Women\", inplace=True, ascending=False)\n",
    "df.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Category</th>\n",
       "      <th>All bios</th>\n",
       "      <th>Men</th>\n",
       "      <th>Women</th>\n",
       "      <th>% Women</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>All categories</td>\n",
       "      <td>848989</td>\n",
       "      <td>718413</td>\n",
       "      <td>130576</td>\n",
       "      <td>15.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>artists</td>\n",
       "      <td>30265</td>\n",
       "      <td>21781</td>\n",
       "      <td>8484</td>\n",
       "      <td>28.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sports</td>\n",
       "      <td>22673</td>\n",
       "      <td>17528</td>\n",
       "      <td>5145</td>\n",
       "      <td>22.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>monarch</td>\n",
       "      <td>12644</td>\n",
       "      <td>10120</td>\n",
       "      <td>2524</td>\n",
       "      <td>20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>scientist</td>\n",
       "      <td>10384</td>\n",
       "      <td>8438</td>\n",
       "      <td>1946</td>\n",
       "      <td>18.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>programming</td>\n",
       "      <td>1047</td>\n",
       "      <td>995</td>\n",
       "      <td>52</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Category  All bios     Men   Women  % Women\n",
       "0  All categories    848989  718413  130576     15.4\n",
       "1         artists     30265   21781    8484     28.0\n",
       "2          sports     22673   17528    5145     22.7\n",
       "3         monarch     12644   10120    2524     20.0\n",
       "4       scientist     10384    8438    1946     18.7\n",
       "5     programming      1047     995      52      5.0"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "toprow = [(\"All categories\", 848989, 718413, 130576, round(130576/848989*100,1))]\n",
    "pd.concat([pd.DataFrame(toprow, columns = list(df)), df], ignore_index=True) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pointwise Mutual Information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PMICalculator:\n",
    "    \n",
    "    def __init__(self, counter1, counter2, n1, n2):\n",
    "        self.counter1 = counter1\n",
    "        self.counter2 = counter2\n",
    "        self.n1 = n1\n",
    "        self.n2 = n2\n",
    "        self.lpc1 = log(n1 / (n1 + n2))\n",
    "\n",
    "    def pmi(self, word):\n",
    "        try:\n",
    "            return (log(self.counter1[word]) - \n",
    "            self.lpc1 -\n",
    "            log(self.counter1[word] + self.counter2[word]))\n",
    "        except ValueError:\n",
    "            return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "pc_sports = PMICalculator(female_sports, male_sports, 5145, 17528)\n",
    "pc_monarch = PMICalculator(female_monarch, male_monarch, 2524, 10120)\n",
    "pc_scientist = PMICalculator(female_scientist, male_scientist, 1946, 8438)\n",
    "pc_artists = PMICalculator(female_artists, male_artists, 8484, 21781)\n",
    "pc_programming = PMICalculator(female_programming, male_programming, 52, 995)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "sports_bigrams = male_sports + female_sports\n",
    "top_sports = sports_bigrams.most_common(20000)\n",
    "top_sports_with_pmi = [(p[0], p[1], pc_sports.pmi(p[0])) for p in top_sports]\n",
    "top_sports_with_pmi.sort(key = lambda x: x[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-3.3330917361095396"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pc_sports.pmi((\"male\", \"athlete\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.460421168880937"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pc_sports.pmi((\"female\", \"athlete\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "247"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sports_bigrams[(\"male\", \"athlete\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "356"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sports_bigrams[(\"female\", \"athlete\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(('town', 'f'), 4762, -6.985273607088316),\n",
       " (('men', 'footer'), 3020, -6.529862690410423),\n",
       " (('he', 'became'), 2847, -6.470871667319544),\n",
       " (('championships', 'men'), 8485, -6.464293469041724),\n",
       " (('league', 'one'), 2683, -6.411541430467131),\n",
       " (('he', 'did'), 2153, -6.19146807740587),\n",
       " (('cfb', 'yearly'), 2126, -6.1788481389434),\n",
       " (('year', 'he'), 2079, -6.15649286789952),\n",
       " (('his', 'personal'), 1957, -6.096018547437583),\n",
       " (('united', 'f'), 7706, -6.08031074410772),\n",
       " (('he', 'played'), 7609, -6.06764325509814),\n",
       " (('he', 'won'), 9484, -6.064774114792703),\n",
       " (('paralympics', 'men'), 1836, -6.032195151221943),\n",
       " (('la', 'liga'), 3628, -6.0201402107165904),\n",
       " (('athletics', 'men'), 6987, -5.9823627774740675),\n",
       " (('ham', 'united'), 1736, -5.976189475261803),\n",
       " (('ref', 'his'), 3413, -5.959050348388269),\n",
       " (('he', 'started'), 1701, -5.955822172437369),\n",
       " (('he', 'went'), 1615, -5.903940815698264),\n",
       " (('his', 'first'), 14486, -5.90056394746056),\n",
       " (('games', 'he'), 1590, -5.8883398752557845),\n",
       " (('won', 'his'), 1567, -5.873768822397528),\n",
       " (('his', 'team'), 1565, -5.8724916830157605),\n",
       " (('park', 'rangers'), 1417, -5.773147819732188),\n",
       " (('later', 'he'), 1375, -5.743059590142178),\n",
       " (('american', 'male'), 2746, -5.741603985809478),\n",
       " (('his', 'last'), 1372, -5.740875388327337),\n",
       " (('states', 'men'), 1367, -5.737224416765456),\n",
       " (('football', 'season'), 1327, -5.707526614373714),\n",
       " (('his', 'professional'), 1285, -5.675364577370828),\n",
       " (('strongest', 'man'), 2546, -5.665982178598915),\n",
       " (('championships', 'he'), 1218, -5.62181602831135),\n",
       " (('rangers', 'f'), 1191, -5.599399149396807),\n",
       " (('no', 'cfb'), 1189, -5.597718476732289),\n",
       " (('1908', 'summer'), 1188, -5.596877079964098),\n",
       " (('the', '1908'), 1181, -5.590967396238869),\n",
       " (('ref', 'he'), 22433, -5.590699968888372),\n",
       " (('2008', 'he'), 1180, -5.590120297501217),\n",
       " (('light', 'heavyweight'), 1175, -5.5858740066197665),\n",
       " (('his', 'career'), 5853, -5.582122296898627),\n",
       " (('2009', 'he'), 1167, -5.579042212328063),\n",
       " (('real', 'madrid'), 2262, -5.547708056157628),\n",
       " (('games', 'men'), 2228, -5.532563000528738),\n",
       " (('nfl', 'com'), 1079, -5.500640545299642),\n",
       " (('his', 'third'), 1075, -5.49692652060327),\n",
       " (('wolverhampton', 'wanderers'), 1072, -5.494131921672254),\n",
       " (('he', 'then'), 2087, -5.467186306558762),\n",
       " (('confederations', 'cup'), 1035, -5.459007285740976),\n",
       " (('junior', 'ice'), 1016, -5.4404792081799345),\n",
       " (('west', 'ham'), 2874, -5.381698358012368)]"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Men\n",
    "top_sports_with_pmi[:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(('english', 'female'), 219, 1.483149419958493),\n",
       " (('the', 'lpga'), 219, 1.483149419958493),\n",
       " (('cup', 'women'), 217, 1.483149419958493),\n",
       " (('events', 'she'), 214, 1.483149419958493),\n",
       " (('she', 'beat'), 214, 1.483149419958493),\n",
       " (('open', 'she'), 211, 1.483149419958493),\n",
       " (('australian', 'female'), 209, 1.483149419958493),\n",
       " (('she', 'participated'), 209, 1.483149419958493),\n",
       " (('york', 'flash'), 209, 1.483149419958493),\n",
       " (('blue', 'fc'), 207, 1.483149419958493),\n",
       " (('she', 'graduated'), 204, 1.483149419958493),\n",
       " (('female', 'tennis'), 204, 1.483149419958493),\n",
       " (('she', 'improved'), 201, 1.483149419958493),\n",
       " (('marathon', 'she'), 201, 1.483149419958493),\n",
       " (('reign', 'fc'), 194, 1.483149419958493),\n",
       " (('2014', 'she'), 193, 1.483149419958493),\n",
       " (('2004', 'she'), 193, 1.483149419958493),\n",
       " (('began', 'her'), 190, 1.483149419958493),\n",
       " (('her', 'heat'), 190, 1.483149419958493),\n",
       " (('athletics', 'she'), 189, 1.483149419958493),\n",
       " (('results', 'women'), 188, 1.483149419958493),\n",
       " (('she', 'retired'), 185, 1.483149419958493),\n",
       " (('2005', 'she'), 183, 1.483149419958493),\n",
       " (('thorns', 'fc'), 183, 1.483149419958493),\n",
       " (('she', 'entered'), 182, 1.483149419958493),\n",
       " (('wnba', 'season'), 180, 1.483149419958493),\n",
       " (('fc', 'kansas'), 179, 1.483149419958493),\n",
       " (('euro', '2013'), 179, 1.483149419958493),\n",
       " (('she', 'helped'), 178, 1.483149419958493),\n",
       " (('iihf', 'women'), 178, 1.483149419958493),\n",
       " (('addfad', 'style'), 177, 1.483149419958493),\n",
       " (('her', 'olympic'), 176, 1.483149419958493),\n",
       " (('female', 'biathletes'), 175, 1.483149419958493),\n",
       " (('m', 'she'), 173, 1.483149419958493),\n",
       " (('throw', 'women'), 171, 1.483149419958493),\n",
       " (('2002', 'she'), 171, 1.483149419958493),\n",
       " (('f0f8ff', 'style'), 171, 1.483149419958493),\n",
       " (('category', 'lesbian'), 170, 1.483149419958493),\n",
       " (('she', 'joined'), 169, 1.483149419958493),\n",
       " (('set', 'her'), 168, 1.483149419958493),\n",
       " (('fa', 'wsl'), 168, 1.483149419958493),\n",
       " (('female', 'track'), 167, 1.483149419958493),\n",
       " (('her', 'teammates'), 166, 1.483149419958493),\n",
       " (('sortfbs', '1'), 166, 1.483149419958493),\n",
       " (('hurdles', '100'), 165, 1.483149419958493),\n",
       " (('international', 'women'), 164, 1.483149419958493),\n",
       " (('2000', 'she'), 162, 1.483149419958493),\n",
       " (('retired', 'female'), 162, 1.483149419958493),\n",
       " (('2016', 'she'), 162, 1.483149419958493),\n",
       " (('competition', 'she'), 162, 1.483149419958493)]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Women\n",
    "top_sports_with_pmi[-50:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Monarch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "monarch_bigrams = male_monarch + female_monarch\n",
    "top_monarch = monarch_bigrams.most_common(20000)\n",
    "top_monarch_with_pmi = [(p[0], p[1], pc_monarch.pmi(p[0])) for p in top_monarch]\n",
    "top_monarch_with_pmi.sort(key = lambda x: x[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(('american', 'football'), 4467, -6.793134471677714),\n",
       " (('united', 'f'), 3335, -6.5008901086753195),\n",
       " (('cricket', 'club'), 2072, -6.024931753704969),\n",
       " (('football', 'league'), 5673, -5.933523218984052),\n",
       " (('county', 'cricket'), 1720, -5.838741720133094),\n",
       " (('english', 'football'), 1683, -5.8169953445164015),\n",
       " (('league', 'team'), 1502, -5.703214982649675),\n",
       " (('league', 'players'), 1435, -5.6575822785193175),\n",
       " (('hi', 'football'), 1412, -5.641424568378783),\n",
       " (('national', 'league'), 1350, -5.596522021758071),\n",
       " (('0', 'colspan'), 1328, -5.5800914803619746),\n",
       " (('sport', 'date'), 1296, -5.555700027237815),\n",
       " (('com', 'players'), 1285, -5.547176147654916),\n",
       " (('he', 'played'), 3389, -5.4183400332645375),\n",
       " (('f', 'c'), 23118, -5.2589751338318615),\n",
       " (('free', 'agent'), 857, -5.142100068923376),\n",
       " (('football', 'teams'), 853, -5.137421697817275),\n",
       " (('major', 'league'), 1619, -5.085078923443286),\n",
       " (('league', 'baseball'), 1581, -5.061327806975123),\n",
       " (('league', 'season'), 736, -4.989892269054472),\n",
       " (('managers', 'category'), 735, -4.988532649538432),\n",
       " (('sport', 'football'), 734, -4.987171178940111),\n",
       " (('football', 'coach'), 704, -4.945440506483638),\n",
       " (('fa', 'cup'), 684, -4.916620067948146),\n",
       " (('regular', 'season'), 660, -4.880901985346067),\n",
       " (('national', 'rugby'), 1269, -4.841499437480038),\n",
       " (('career', 'he'), 607, -4.797190941385094),\n",
       " (('ncaa', 'division'), 1184, -4.772168785209601),\n",
       " (('sport', 'accessdate'), 572, -4.737801141705393),\n",
       " (('statistics', 'at'), 535, -4.670928897221602)]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Men\n",
    "top_monarch_with_pmi[:30]\n",
    "\n",
    "# I THINK THERE IS AN ERROR HERE, as the bigrams seem to indicate the sports category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(('list', 'a'), 1032, -4.634768915807158),\n",
       " (('footballer', 'who'), 509, -4.621110166876118),\n",
       " (('season', 'he'), 1003, -4.606265757727586),\n",
       " (('team', 'all'), 998, -4.601268246077114),\n",
       " (('his', 'debut'), 961, -4.563489378735943),\n",
       " (('he', 'finished'), 478, -4.558272882817052),\n",
       " (('appearances', 'for'), 462, -4.524227041407334),\n",
       " (('football', 'player'), 452, -4.502344330157827),\n",
       " (('football', 'footballer'), 446, -4.48898110234566),\n",
       " (('he', 'graduated'), 427, -4.44544616355422),\n",
       " (('he', 'studied'), 426, -4.4431014965949664),\n",
       " (('season', 'with'), 838, -4.426533070247733),\n",
       " (('first', 'team'), 1579, -4.366914803461347),\n",
       " (('uk', 'sport1'), 1552, -4.349667489948869),\n",
       " (('all', 'american'), 776, -4.349667489948869),\n",
       " (('league', 'championship'), 388, -4.349667489948869),\n",
       " (('sport1', 'hi'), 1549, -4.347732629622573),\n",
       " (('football', 'eng'), 384, -4.3393047029133225),\n",
       " (('second', 'team'), 377, -4.3209073377736065),\n",
       " (('http', 'sports'), 370, -4.302165155963865),\n",
       " (('season', 'as'), 367, -4.294023998380166),\n",
       " (('category', 'male'), 365, -4.288559503908087),\n",
       " (('category', 'st'), 365, -4.288559503908087),\n",
       " (('male', 'actors'), 361, -4.277540108658476),\n",
       " (('football', 'at'), 711, -4.262187399568891),\n",
       " (('sports', 'espn'), 355, -4.2607799398010116),\n",
       " (('coach', 'for'), 353, -4.255130207258892),\n",
       " (('2007', 'he'), 352, -4.252293325923692),\n",
       " (('goal', 'in'), 349, -4.243734072528023),\n",
       " (('2006', 'he'), 346, -4.23510092538332),\n",
       " (('re', 'signed'), 343, -4.226392597491535),\n",
       " (('2011', 'he'), 342, -4.2234728873882),\n",
       " (('weight', 'lbs'), 675, -4.21022766063818),\n",
       " (('signed', 'for'), 667, -4.198305015681274),\n",
       " (('bbc', 'sport'), 2276, -4.172926796516504),\n",
       " (('championship', 'game'), 325, -4.172487332655333),\n",
       " (('height', 'in'), 972, -4.169405666117925),\n",
       " (('city', 'f'), 1939, -4.166830336340192),\n",
       " (('cambridge', 'united'), 320, -4.156983146119368),\n",
       " (('a', 'f'), 1878, -4.1348653408657485),\n",
       " (('challenge', 'cup'), 932, -4.1273826763430765),\n",
       " (('the', 'football'), 926, -4.120924096303665),\n",
       " (('premier', 'league'), 1233, -4.119573292370049),\n",
       " (('leeds', 'united'), 306, -4.112247252277976),\n",
       " (('rangers', 'f'), 301, -4.095772415074471),\n",
       " (('baseball', 'biography'), 299, -4.0891057237162824),\n",
       " (('com', 'player'), 298, -4.085755636831),\n",
       " (('2012', 'he'), 296, -4.079021604649656),\n",
       " (('training', 'camp'), 294, -4.072241917664277),\n",
       " (('a', 'player'), 290, -4.058543073306115),\n",
       " (('general', 'staff'), 290, -4.058543073306115),\n",
       " (('league', 'division'), 289, -4.055088838438028),\n",
       " (('birmingham', 'city'), 287, -4.048144366085217),\n",
       " (('he', 'earned'), 285, -4.041151330594246),\n",
       " (('on', 'loan'), 846, -4.0305692212637085),\n",
       " (('cup', 'final'), 1112, -4.0162832640162325),\n",
       " (('career', 'high'), 278, -4.0162832640162325),\n",
       " (('britain', 'national'), 278, -4.0162832640162325),\n",
       " (('season', '2014'), 272, -3.9944642166215933),\n",
       " (('reference', 'com'), 1630, -3.9932379748983484),\n",
       " (('playing', 'career'), 538, -3.9833735299274347),\n",
       " (('cricketers', 'category'), 1344, -3.98262975897459),\n",
       " (('the', 'fa'), 268, -3.979649130836452),\n",
       " (('lang', 'zh'), 526, -3.9608161825033603),\n",
       " (('dallas', 'cowboys'), 525, -3.9589132323572738),\n",
       " (('was', 'waived'), 262, -3.957006654086692),\n",
       " (('coach', 'in'), 262, -3.957006654086692),\n",
       " (('name', 'games'), 522, -3.9531825576482893),\n",
       " (('players', 'player'), 522, -3.9531825576482893),\n",
       " (('personnel', 'of'), 519, -3.9474188529315386)]"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_monarch_with_pmi[30:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(('hedvig', 'taube'), 117, 1.576551733588989),\n",
       " (('throughout', 'her'), 118, 1.576851673603235),\n",
       " (('diana', 's'), 125, 1.5788146579688442),\n",
       " (('best', 'actress'), 218, 1.5834290615573279),\n",
       " (('danish', 'princesses'), 161, 1.586179290038249),\n",
       " (('joan', 'sutherland'), 125, 1.5870451571053596),\n",
       " (('actresses', 'category'), 384, 1.5902844404765721),\n",
       " (('she', 'appeared'), 192, 1.5902844404765721),\n",
       " (('harvnb', 'chibnall'), 145, 1.590431164855091),\n",
       " (('weir', '2008'), 157, 1.5920446467397262),\n",
       " (('ref', 'porter'), 161, 1.5925285177169082),\n",
       " (('dame', 'grand'), 375, 1.5952084677445209),\n",
       " (('her', 'spouse'), 328, 1.5990677570825893),\n",
       " (('women', 'writers'), 192, 1.6113378496744044),\n",
       " (('she', 'attended'), 186, 1.6113378496744044),\n",
       " (('chibnall', '1991'), 151, 1.6113378496744044),\n",
       " (('princess', 'diana'), 145, 1.6113378496744044),\n",
       " (('der', 'maur'), 143, 1.6113378496744044),\n",
       " (('martha', 'stewart'), 139, 1.6113378496744044),\n",
       " (('valide', 'sultan'), 139, 1.6113378496744044),\n",
       " (('category', 'actresses'), 136, 1.6113378496744044),\n",
       " (('atkinson', 'emma'), 133, 1.6113378496744044),\n",
       " (('emma', 'willsher'), 133, 1.6113378496744044),\n",
       " (('fraser', '1994'), 128, 1.6113378496744044),\n",
       " (('name', 'atkinson'), 127, 1.6113378496744044),\n",
       " (('sfn', 'gelardi'), 121, 1.6113378496744044),\n",
       " (('gelardi', '2005'), 120, 1.6113378496744044),\n",
       " (('she', 'visited'), 119, 1.6113378496744044),\n",
       " (('raja', 'permaisuri'), 117, 1.6113378496744044),\n",
       " (('medrano', '2004'), 115, 1.6113378496744044)]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Women\n",
    "top_monarch_with_pmi[-30:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scientist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "scientist_bigrams = male_scientist + female_scientist\n",
    "top_scientist = scientist_bigrams.most_common(20000)\n",
    "top_scientist_with_pmi = [(p[0], p[1], pc_scientist.pmi(p[0])) for p in top_scientist]\n",
    "top_scientist_with_pmi.sort(key = lambda x: x[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Men\n",
    "top_scientist_with_pmi[:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Women\n",
    "top_scientist_with_pmi[-30:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2274"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scientist_bigrams[(\"women\", \"scientists\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scientist_bigrams[(\"men\", \"scientists\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scientist_bigrams[(\"female\", \"scientist\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.30688794809018294\n"
     ]
    }
   ],
   "source": [
    "print(pc_scientist.pmi((\"first\", \"person\")))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Artists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "artists_bigrams = male_artists + female_artists\n",
    "top_artists = artists_bigrams.most_common(20000)\n",
    "top_artists_with_pmi = [(p[0], p[1], pc_artists.pmi(p[0])) for p in top_artists]\n",
    "top_artists_with_pmi.sort(key = lambda x: x[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Men\n",
    "top_artists_with_pmi[:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Women\n",
    "top_artists_with_pmi[-30:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Programming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "programming_bigrams = male_programming + female_programming\n",
    "top_programming = programming_bigrams.most_common(20000)\n",
    "top_programming_with_pmi = [(p[0], p[1], pc_programming.pmi(p[0])) for p in top_programming]\n",
    "top_programming_with_pmi.sort(key = lambda x: x[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(('which', 'he'), 314, -2.7469524936191436),\n",
       " (('where', 'he'), 613, -2.7227772630871567),\n",
       " (('he', 'was'), 1619, -2.595363992160471),\n",
       " (('his', 'first'), 253, -2.530948996438411),\n",
       " (('his', 'work'), 248, -2.510988253875873),\n",
       " (('archiveurl', 'http'), 187, -2.2286681245654774),\n",
       " (('businesspeople', 'in'), 177, -2.1737092402847193),\n",
       " (('in', 'may'), 166, -2.109547296067434),\n",
       " (('acquired', 'by'), 165, -2.103504981611471),\n",
       " (('chairman', 'of'), 163, -2.091309708517653),\n",
       " (('webcitation', 'org'), 159, -2.066463709931122),\n",
       " (('apple', 'ii'), 159, -2.066463709931122),\n",
       " (('www', 'webcitation'), 158, -2.0601545407378574),\n",
       " (('the', 'apple'), 158, -2.0601545407378574),\n",
       " (('operating', 'system'), 297, -1.998144465953645),\n",
       " (('software', 'category'), 145, -1.9742932501314647),\n",
       " (('apple', 'inc'), 139, -1.932033440841582),\n",
       " (('nbsp', 'million'), 138, -1.9248131928680956),\n",
       " (('publisher', 'ref'), 136, -1.9102143934469429),\n",
       " (('on', 'his'), 262, -1.8727568309120417),\n",
       " (('df', 'mdy'), 131, -1.8727568309120417),\n",
       " (('a', 'company'), 126, -1.8338414146623685),\n",
       " (('for', 'his'), 499, -1.823871242342519),\n",
       " (('23', 'ref'), 124, -1.8178410733159276),\n",
       " (('million', 'to'), 123, -1.8097438630833076),\n",
       " (('isbn', '1'), 123, -1.8097438630833076),\n",
       " (('stanford', 'edu'), 119, -1.7766830008224201),\n",
       " (('he', 'and'), 117, -1.7597334425086468),\n",
       " (('company', 'founders'), 117, -1.7597334425086468),\n",
       " (('august', '2012'), 116, -1.751149698817255)]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Men\n",
    "top_programming_with_pmi[:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(('kay', 'mcnulty'), 12, 3.0024404922891095),\n",
       " (('abdul', 'karim'), 12, 3.0024404922891095),\n",
       " (('mayer', 'was'), 12, 3.0024404922891095),\n",
       " (('stormy', 'peters'), 12, 3.0024404922891095),\n",
       " (('new', 'justice'), 12, 3.0024404922891095),\n",
       " (('category', 'women'), 67, 3.00244049228911),\n",
       " (('ada', 'lovelace'), 62, 3.00244049228911),\n",
       " (('mm', 'b'), 31, 3.00244049228911),\n",
       " (('of', 'lovelace'), 18, 3.00244049228911),\n",
       " (('lord', 'byron'), 18, 3.00244049228911),\n",
       " (('women', 'scientists'), 18, 3.00244049228911),\n",
       " (('justice', 'minister'), 18, 3.00244049228911),\n",
       " (('to', 'governability'), 18, 3.00244049228911),\n",
       " (('lovelace', 's'), 17, 3.00244049228911),\n",
       " (('last', 'conway'), 17, 3.00244049228911),\n",
       " (('smita', 'bellur'), 17, 3.00244049228911),\n",
       " (('1972', 'p'), 10, 3.00244049228911),\n",
       " (('hopper', 's'), 10, 3.00244049228911),\n",
       " (('center', 'center'), 10, 3.00244049228911),\n",
       " (('transsexual', 'women'), 10, 3.00244049228911),\n",
       " (('gender', 'identity'), 10, 3.00244049228911),\n",
       " (('maxwell', 'medal'), 10, 3.00244049228911),\n",
       " (('audrey', 'tang'), 10, 3.00244049228911),\n",
       " (('qing', 'history'), 10, 3.00244049228911),\n",
       " (('eniac', 'was'), 10, 3.00244049228911),\n",
       " (('technology', 'park'), 10, 3.00244049228911),\n",
       " (('jennings', 'bartik'), 10, 3.00244049228911),\n",
       " (('vintagecomputing', 'com'), 10, 3.00244049228911),\n",
       " (('deccan', 'herald'), 10, 3.00244049228911),\n",
       " (('article', 'shaked'), 10, 3.00244049228911)]"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Women\n",
    "top_programming_with_pmi[-30:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8151d42f8bbc45c6be2430b202bdbb00",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, max=850399.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8c887c37f7e14b7e87cdd0ec465bcb96",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "IntText(value=0, description='Article Count:')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Unigram categories:\n",
    "\n",
    "master_dict = {\n",
    "    'male_sports_u': collections.Counter(), \n",
    "    'female_sports_u': collections.Counter(),\n",
    "    'male_monarch_u': collections.Counter(), \n",
    "    'female_monarch_u': collections.Counter(),\n",
    "    'male_artists_u': collections.Counter(), \n",
    "    'female_artists_u': collections.Counter(),\n",
    "    'male_scientist_u': collections.Counter(), \n",
    "    'female_scientist_u': collections.Counter(),\n",
    "    'male_programming_u': collections.Counter(), \n",
    "    'female_programming_u': collections.Counter() }\n",
    "cats = ['sports', 'monarch', 'artists', 'scientist', 'programming']\n",
    "\n",
    "fp = FloatProgress(min=0, max=850399)\n",
    "all_articles = IntText(value=0, description=\"Article Count:\")\n",
    "display(fp, all_articles)\n",
    "article_count = 0\n",
    "start = time.time()\n",
    "\n",
    "try:\n",
    "    with open(\"data/gendered-labeled-articles.stripped\", 'r') as f:\n",
    "        for line in f:\n",
    "            article_count += 1\n",
    "            all_articles.value += 1\n",
    "            if article_count % 1000 == 0:\n",
    "                fp.value = article_count\n",
    "            words = line.split()\n",
    "            art_id = words.pop(0)\n",
    "            if int(art_id) in all_ids:\n",
    "                male = is_male[art_id]\n",
    "                for c in cats:\n",
    "                    if int(art_id) in topic_to_articles[c]:\n",
    "                        if male:\n",
    "                            master_dict[\"male_\" + c + \"_u\"].update(words)\n",
    "                        else:\n",
    "                            master_dict[\"female_\" + c + \"_u\"].update(words)\n",
    "finally:\n",
    "    print(\"Articles:\", article_count, \" Time:\", (time.time() - start), \"seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SAVE\n",
    "for c in cats:\n",
    "    save_object(master_dict[\"female_\" + c + \"_u\"], \n",
    "                'data/bigrams/female_{}_u.pkl'.format(c))\n",
    "    save_object(master_dict[\"male_\" + c + \"_u\"], \n",
    "                'data/bigrams/male_{}_u.pkl'.format(c))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12296493"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(master_dict[\"male_scientist_u\"].values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2840901"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(master_dict[\"female_scientist_u\"].values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38561387"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(master_dict[\"male_artists_u\"].values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16363597"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(master_dict[\"female_artists_u\"].values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "pc_artists2 = PMICalculator(master_dict[\"female_artists_u\"], \n",
    "                            master_dict[\"male_artists_u\"], 8484, 21781)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "pc_scientist2 = PMICalculator(master_dict[\"female_scientist_u\"], \n",
    "                              master_dict[\"male_scientist_u\"], 1946, 8438)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "pc_monarch2 = PMICalculator(master_dict[\"female_monarch_u\"], \n",
    "                              master_dict[\"male_monarch_u\"], 2524, 10120)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('nfl', 14983, -7.310186473808795),\n",
       " ('wigan', 2124, -6.049718532687425),\n",
       " ('quarterback', 1959, -5.968851568270137),\n",
       " ('batting', 1459, -5.674168698848381),\n",
       " ('rovers', 1390, -5.625721176450333),\n",
       " ('rams', 951, -5.246176212870986),\n",
       " ('tampa', 801, -5.074523097393955),\n",
       " ('rookie', 801, -5.074523097393955),\n",
       " ('striker', 761, -5.023295508187282),\n",
       " ('innings', 732, -4.984442664286907),\n",
       " ('touchdown', 1455, -4.9782761493712435),\n",
       " ('interception', 702, -4.942595554351406),\n",
       " ('brentford', 691, -4.926801974093266),\n",
       " ('giants', 1314, -4.876346168810206),\n",
       " ('bengals', 622, -4.821602243064775),\n",
       " ('goryeo', 615, -4.810284418132113),\n",
       " ('packers', 1223, -4.804577105452823),\n",
       " ('tackles', 1826, -4.799932922812399),\n",
       " ('ravens', 563, -4.721941778465286),\n",
       " ('goals6', 558, -4.713021112706906),\n",
       " ('arsenal', 532, -4.66530563966724),\n",
       " ('100s', 508, -4.619143597904078),\n",
       " ('cricket', 6999, -4.603127381399876),\n",
       " ('cincinnati', 1499, -4.602603359760083),\n",
       " ('behr', 496, -4.5952380770505235),\n",
       " ('nero', 489, -4.581024639800468),\n",
       " ('mlb', 1437, -4.5603627477365105),\n",
       " ('caps6', 479, -4.5603627477365105),\n",
       " ('goals7', 470, -4.5413948450297),\n",
       " ('football', 37977, -4.514557534833935),\n",
       " ('yards', 4499, -4.497687486172722),\n",
       " ('vikings', 11180, -4.491668072166485),\n",
       " ('starter', 436, -4.46630439367463),\n",
       " ('dagenham', 432, -4.457087738569706),\n",
       " ('falcons', 863, -4.455929660849078),\n",
       " ('baseball', 7754, -4.454254510660305),\n",
       " ('receiver', 853, -4.4442745172573295),\n",
       " ('sarkozy', 423, -4.436034329371873),\n",
       " ('conf', 423, -4.436034329371873),\n",
       " ('caps7', 418, -4.424143582850352),\n",
       " ('cagney', 415, -4.4169406705562935),\n",
       " ('diefenbaker', 400, -4.380126697433577),\n",
       " ('sport1', 1588, -4.3725984310127854),\n",
       " ('stravinsky', 380, -4.328833403046027),\n",
       " ('husayn', 379, -4.326198355408022),\n",
       " ('goals8', 377, -4.3209073377736065),\n",
       " ('stevenage', 749, -4.314253953282869),\n",
       " ('nimoy', 365, -4.288559503908087),\n",
       " ('hyder', 361, -4.277540108658476),\n",
       " ('chirac', 361, -4.277540108658476)]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "monarch_unigrams = master_dict[\"male_monarch_u\"] + master_dict[\"female_monarch_u\"]\n",
    "top_monarch2 = monarch_unigrams.most_common(20000)\n",
    "top_monarch2_with_pmi = [(p[0], p[1], pc_monarch2.pmi(p[0])) for p in top_monarch2]\n",
    "top_monarch2_with_pmi.sort(key = lambda x: x[-1])\n",
    "# Men\n",
    "top_monarch2_with_pmi[:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('mensch', 148, 1.5699526335115506),\n",
       " ('wahlström', 128, 1.5714919411272046),\n",
       " ('widstrand', 128, 1.5714919411272046),\n",
       " ('aubyn', 104, 1.5721171365211237),\n",
       " ('hämtad', 115, 1.5759359226234881),\n",
       " ('platten', 90, 1.577436297998723),\n",
       " ('azaria', 66, 1.5805661910076507),\n",
       " ('conroy', 169, 1.5813055625755297),\n",
       " ('mindy', 107, 1.5828999143538711),\n",
       " ('woodham', 73, 1.583558285567329),\n",
       " ('permaisuri', 159, 1.5858587643734197),\n",
       " ('plowden', 87, 1.5880809875101374),\n",
       " ('brava', 135, 1.5888649938223454),\n",
       " ('ampuan', 97, 1.5905037627715624),\n",
       " ('sridevi', 297, 1.5909289780431974),\n",
       " ('mirabella', 99, 1.5909289780431974),\n",
       " ('falle', 120, 1.5945307313580237),\n",
       " ('valide', 188, 1.5952517119227796),\n",
       " ('gelardi', 188, 1.5952517119227796),\n",
       " ('tamannaah', 128, 1.5955894927062655),\n",
       " ('kvindebiografisk', 66, 1.596070377543616),\n",
       " ('brunhilda', 66, 1.596070377543616),\n",
       " ('kunglig', 67, 1.5962999723098639),\n",
       " ('titmuss', 74, 1.5977321976186252),\n",
       " ('devyani', 102, 1.6014855532313934),\n",
       " ('christabel', 104, 1.6016759387626678),\n",
       " ('moyet', 116, 1.60267978693129),\n",
       " ('charlize', 122, 1.603107350537889),\n",
       " ('abramović', 165, 1.6052588035980229),\n",
       " ('theron', 173, 1.6055407319900779),\n",
       " ('woodhull', 174, 1.6055741449576546),\n",
       " ('braxton', 190, 1.6060607925735608),\n",
       " ('highsmith', 191, 1.6113378496744044),\n",
       " ('khobragade', 159, 1.6113378496744044),\n",
       " ('kaling', 125, 1.6113378496744044),\n",
       " ('hatoum', 122, 1.6113378496744044),\n",
       " ('pinkham', 107, 1.6113378496744044),\n",
       " ('kösem', 94, 1.6113378496744044),\n",
       " ('chessbase', 94, 1.6113378496744044),\n",
       " ('copps', 88, 1.6113378496744044),\n",
       " ('foozie', 85, 1.6113378496744044),\n",
       " ('abramovic', 80, 1.6113378496744044),\n",
       " ('yezierska', 74, 1.6113378496744044),\n",
       " ('tegenborg', 69, 1.6113378496744044),\n",
       " ('falkdalen', 69, 1.6113378496744044),\n",
       " ('queenmargretheorderspage', 68, 1.6113378496744044),\n",
       " ('panti', 68, 1.6113378496744044),\n",
       " ('tegenborg2010', 67, 1.6113378496744044),\n",
       " ('safiye', 67, 1.6113378496744044),\n",
       " ('tpr', 66, 1.6113378496744044)]"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Women\n",
    "top_monarch2_with_pmi[-50:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}