8. correlation_of_mover_scores_combined.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import pandas as pd\n",
    "from scipy.stats import pearsonr, spearmanr\n",
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('seaborn-whitegrid')\n",
    "import numpy as np\n",
    "\n",
    "lang2mono = {'EN': 'bert-base-uncased',\n",
    "             'ID': 'indobert-base-uncased',\n",
    "             'FR': 'camembert-base',\n",
    "             'TR': 'bert-base-turkish-uncased',\n",
    "             'ZH': 'bert-base-chinese',\n",
    "             'RU': 'rubert-base-cased',\n",
    "             'DE': 'bert-base-german-dbmdz-uncased',\n",
    "             'ES': 'bert-base-spanish-wwm-uncased'}\n",
    "\n",
    "LANGS = ['EN', 'ID', 'FR', 'TR', 'ZH', 'RU', 'DE', 'ES']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_human_annotation(lang, types='focus'):\n",
    "    if types=='focus':\n",
    "        path_human = f'mturk/annotation_result/{lang}/human_focus_final.csv'\n",
    "    else:\n",
    "        assert types=='coverage'\n",
    "        path_human = f'mturk/annotation_result/{lang}/human_coverage_final.csv'\n",
    "    human = pd.read_csv(path_human)\n",
    "    return human\n",
    "\n",
    "def align(df, human_score, model):\n",
    "    human = []\n",
    "    machine = []\n",
    "    #print(df)\n",
    "    for idx, row in df.iterrows():\n",
    "        human.append(human_score[(human_score['model'] == model) & (human_score['id']==row['fnames'])]['score'].values[0])\n",
    "        machine.append(row['moverscore'])\n",
    "    return human, machine\n",
    "\n",
    "def read(lang, human_score, pretrained):\n",
    "    path_BERT = f'mover_score/{lang}--BERT--{pretrained}.csv'\n",
    "    path_PG = f'mover_score/{lang}--PG--{pretrained}.csv'\n",
    "        \n",
    "    humans = []; machines = []\n",
    "\n",
    "    df_BERT = pd.read_csv(path_BERT)\n",
    "    human, machine = align(df_BERT, human_score, 'BERT')\n",
    "    humans += human\n",
    "    machines += machine\n",
    "    \n",
    "    df_PG = pd.read_csv(path_PG)\n",
    "    human, machine = align(df_PG, human_score, 'PG')\n",
    "    humans += human\n",
    "    machines += machine\n",
    "    \n",
    "    #print(humans)\n",
    "    #print(machines)\n",
    "    #return spearmanr(humans, machines)[0], len(humans)\n",
    "    return pearsonr(humans, machines)[0], len(humans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Focus\n",
      "EN 270 bert-base-uncased 0.5802248969741917\n",
      "EN 270 bert-base-multilingual-cased 0.5387837105939391\n",
      "EN 270 bert-base-multilingual-uncased 0.5892230639678848\n",
      "EN 270 xlm-roberta-base 0.5259703002291909\n",
      "EN 270 xlm-roberta-large 0.5051756362902975\n",
      "\n",
      "ID 270 indobert-base-uncased 0.6502816035615807\n",
      "ID 270 bert-base-multilingual-cased 0.6753557391772896\n",
      "ID 270 bert-base-multilingual-uncased 0.6894454416099718\n",
      "ID 270 xlm-roberta-base 0.6260939963618947\n",
      "ID 270 xlm-roberta-large 0.5830980758325219\n",
      "\n",
      "FR 270 camembert-base 0.7119766159090477\n",
      "FR 270 bert-base-multilingual-cased 0.765452543428397\n",
      "FR 270 bert-base-multilingual-uncased 0.7748925546498105\n",
      "FR 270 xlm-roberta-base 0.6861086177502986\n",
      "FR 270 xlm-roberta-large 0.6698015012533034\n",
      "\n",
      "TR 270 bert-base-turkish-uncased 0.8154763953267194\n",
      "TR 270 bert-base-multilingual-cased 0.7908396207454944\n",
      "TR 270 bert-base-multilingual-uncased 0.8108987780857521\n",
      "TR 270 xlm-roberta-base 0.8038763730118588\n",
      "TR 270 xlm-roberta-large 0.7892349565370103\n",
      "\n",
      "ZH 270 bert-base-chinese 0.7694956746745489\n",
      "ZH 270 bert-base-multilingual-cased 0.7603103141553085\n",
      "ZH 270 bert-base-multilingual-uncased 0.7620573117723373\n",
      "ZH 270 xlm-roberta-base 0.7122766356123178\n",
      "ZH 270 xlm-roberta-large 0.5722181575963146\n",
      "\n",
      "RU 270 rubert-base-cased 0.48725810377385104\n",
      "RU 270 bert-base-multilingual-cased 0.5974811421258395\n",
      "RU 270 bert-base-multilingual-uncased 0.5956384709644873\n",
      "RU 270 xlm-roberta-base 0.33077418099655126\n",
      "RU 270 xlm-roberta-large 0.27850100339057404\n",
      "\n",
      "DE 270 bert-base-german-dbmdz-uncased 0.8930680930519515\n",
      "DE 270 bert-base-multilingual-cased 0.8803176218928139\n",
      "DE 270 bert-base-multilingual-uncased 0.8845583296766928\n",
      "DE 270 xlm-roberta-base 0.8785161430822046\n",
      "DE 270 xlm-roberta-large 0.8747211551806544\n",
      "\n",
      "ES 270 bert-base-spanish-wwm-uncased 0.5850174370828856\n",
      "ES 270 bert-base-multilingual-cased 0.6316608150278137\n",
      "ES 270 bert-base-multilingual-uncased 0.6695669264209771\n",
      "ES 270 xlm-roberta-base 0.5894417178801084\n",
      "ES 270 xlm-roberta-large 0.5227669037087737\n",
      "\n"
     ]
    }
   ],
   "source": [
    "LANGS = ['EN', 'ID', 'FR', 'TR', 'ZH', 'RU', 'DE', 'ES']\n",
    "\n",
    "# Focus\n",
    "print('Focus')\n",
    "cors = {}\n",
    "for lang in LANGS:\n",
    "    human_score = read_human_annotation(lang, 'focus')\n",
    "\n",
    "    for pretrained in [ lang2mono[lang], 'bert-base-multilingual-cased', 'bert-base-multilingual-uncased', \\\n",
    "                      'xlm-roberta-base', 'xlm-roberta-large']:    \n",
    "        cor, num = read(lang, human_score, pretrained)\n",
    "        print(lang, num, pretrained, cor)    \n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coverage\n",
      "EN 270 bert-base-uncased 0.5942562985333694\n",
      "EN 270 bert-base-multilingual-cased 0.5244621002608667\n",
      "EN 270 bert-base-multilingual-uncased 0.5872798684003329\n",
      "EN 270 xlm-roberta-base 0.5766830654535068\n",
      "EN 270 xlm-roberta-large 0.5497836265813958\n",
      "\n",
      "ID 270 indobert-base-uncased 0.6240390018610549\n",
      "ID 270 bert-base-multilingual-cased 0.6871406004218968\n",
      "ID 270 bert-base-multilingual-uncased 0.6890553084287263\n",
      "ID 270 xlm-roberta-base 0.6237625780863463\n",
      "ID 270 xlm-roberta-large 0.6149855769356989\n",
      "\n",
      "FR 270 camembert-base 0.6713135396295669\n",
      "FR 270 bert-base-multilingual-cased 0.7218578258116559\n",
      "FR 270 bert-base-multilingual-uncased 0.7484409659661108\n",
      "FR 270 xlm-roberta-base 0.6246436497490657\n",
      "FR 270 xlm-roberta-large 0.5867571249185946\n",
      "\n",
      "TR 270 bert-base-turkish-uncased 0.7828815692563675\n",
      "TR 270 bert-base-multilingual-cased 0.753117180414588\n",
      "TR 270 bert-base-multilingual-uncased 0.7688175164975413\n",
      "TR 270 xlm-roberta-base 0.7420722179927229\n",
      "TR 270 xlm-roberta-large 0.7196229657060806\n",
      "\n",
      "ZH 270 bert-base-chinese 0.7709043347639298\n",
      "ZH 270 bert-base-multilingual-cased 0.7528054069663529\n",
      "ZH 270 bert-base-multilingual-uncased 0.7466352254691162\n",
      "ZH 270 xlm-roberta-base 0.6858227986797759\n",
      "ZH 270 xlm-roberta-large 0.5810129780987485\n",
      "\n",
      "RU 270 rubert-base-cased 0.4070984323727256\n",
      "RU 270 bert-base-multilingual-cased 0.49264108077419067\n",
      "RU 270 bert-base-multilingual-uncased 0.5033682320057382\n",
      "RU 270 xlm-roberta-base 0.21816500943265446\n",
      "RU 270 xlm-roberta-large 0.2142879662074152\n",
      "\n",
      "DE 270 bert-base-german-dbmdz-uncased 0.8745717908896702\n",
      "DE 270 bert-base-multilingual-cased 0.8449082541148866\n",
      "DE 270 bert-base-multilingual-uncased 0.8578179159582341\n",
      "DE 270 xlm-roberta-base 0.8531393849124812\n",
      "DE 270 xlm-roberta-large 0.8414002835306331\n",
      "\n",
      "ES 270 bert-base-spanish-wwm-uncased 0.6076255174162569\n",
      "ES 270 bert-base-multilingual-cased 0.6781956284058634\n",
      "ES 270 bert-base-multilingual-uncased 0.6982088659116144\n",
      "ES 270 xlm-roberta-base 0.6410405495602212\n",
      "ES 270 xlm-roberta-large 0.5598783496812725\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Coverage\n",
    "print('Coverage')\n",
    "\n",
    "cors = {}\n",
    "for lang in LANGS:\n",
    "    human_score = read_human_annotation(lang, 'coverage')\n",
    "    for pretrained in [ lang2mono[lang], 'bert-base-multilingual-cased', 'bert-base-multilingual-uncased', \\\n",
    "                      'xlm-roberta-base', 'xlm-roberta-large']: \n",
    "        cor, num = read(lang, human_score, pretrained)\n",
    "        print(lang, num, pretrained, cor)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}