experiment_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building feature2vec\n"
     ]
    }
   ],
   "source": [
    "import numpy as np \n",
    "from Feature2Vec import Feature2Vec \n",
    "from PLSR import PLSR\n",
    "from utils import * \n",
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"-1\"    \n",
    "\n",
    "SEED = 42\n",
    "np.random.seed(seed = SEED)\n",
    "\n",
    "# path = 'data/mcrae_feature_matrix.csv'\n",
    "path = 'data/cslb_feature_matrix.csv'\n",
    "\n",
    "print('Building feature2vec')\n",
    "model = Feature2Vec(path = path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get state needed to reproduce experiments from paper \n",
    "import pickle as pkl \n",
    "\n",
    "with open('state_zero.pkl', 'rb') as f:\n",
    "    st0 = pkl.load(f)\n",
    "np.random.set_state(st0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "shuffle = np.random.permutation(len(model.concepts))\n",
    "train_concepts = list(np.asarray(model.concepts)[shuffle][:500])\n",
    "test_concepts = list(np.asarray(model.concepts)[shuffle][500:])\n",
    "model.set_vocabulary(train_words = train_concepts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training feature2vec\n",
      "Epoch: 19 Loss: 0.004729189370919474\n"
     ]
    }
   ],
   "source": [
    "print('Training feature2vec')\n",
    "model.train(verbose = 1, epochs = 20, lr = 5e-3, negative_samples = 20)\n",
    "print('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Example features learned for word: dog\n",
      "[['0.25640281976339574' 'has_a_tail']\n",
      " ['0.23381005781910313' 'has_four_legs']\n",
      " ['0.2329559209030151' 'is_an_animal']\n",
      " ['0.22442036094359183' 'is_a_mammal']\n",
      " ['0.2197005260302991' 'has_fur_hair']\n",
      " ['0.2193652359049925' 'has_legs']\n",
      " ['0.21557504271842726' \"is_man's_best_friend\"]\n",
      " ['0.20636315741574815' 'has_paws']\n",
      " ['0.2042316792609977' 'is_small']\n",
      " ['0.20358028406688727' 'is_a_companion']]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# test for word dog\n",
    "word = 'dog'\n",
    "print('Example features learned for word:', word)\n",
    "print(model.top_features(model.wvector(word), top = 10))\n",
    "print('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building partial least squared regression (50)\n",
      "Building partial least squared regression (120)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/40106170/miniconda3/envs/rep_nlm/lib/python3.6/site-packages/sklearn/cross_decomposition/pls_.py:79: ConvergenceWarning: Maximum number of iterations reached\n",
      "  ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# build baseline model (50 and 200)\n",
    "from PLSR import PLSR\n",
    "print('Building partial least squared regression (50)')\n",
    "plsr50 = PLSR(path = path)\n",
    "plsr50.set_vocabulary(train_concepts)\n",
    "plsr50.train(embedding_size = 50)\n",
    "\n",
    "print('Building partial least squared regression (120)')\n",
    "plsr120 = PLSR(path = path)\n",
    "plsr120.set_vocabulary(train_concepts)\n",
    "plsr120.train(embedding_size = 120)\n",
    "print('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "concept_dict_plsr50 = {}\n",
    "for index, concept in enumerate(plsr50.test_words):\n",
    "    concept_dict_plsr50[concept] = plsr50.test_preds[index,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "concept_dict_plsr120 = {}\n",
    "for index, concept in enumerate(plsr120.test_words):\n",
    "    concept_dict_plsr120[concept] = plsr120.test_preds[index,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PLSR 50 neighbour scores\n",
      "Top 1 2.898550724637681\n",
      "Top 5 23.18840579710145\n",
      "Top 10 44.20289855072464\n",
      "Top 20 60.86956521739131\n"
     ]
    }
   ],
   "source": [
    "print('PLSR 50 neighbour scores')\n",
    "tops = [1, 5, 10, 20]\n",
    "for n in tops:\n",
    "    print('Top', n, neighbour_score(concept_dict_plsr50, plsr50, top = n))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PLSR 120 neighbour scores\n",
      "Top 1 7.246376811594203\n",
      "Top 5 34.78260869565217\n",
      "Top 10 55.79710144927537\n",
      "Top 20 71.73913043478261\n"
     ]
    }
   ],
   "source": [
    "print('PLSR 120 neighbour scores')\n",
    "tops = [1, 5, 10, 20]\n",
    "for n in tops:\n",
    "    print('Top', n, neighbour_score(concept_dict_plsr120, plsr120, top = n))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "concept_dict_f2v = {}\n",
    "for index, concept in enumerate(model.test_words):\n",
    "    concept_dict_f2v[concept] = construct_vector(concept, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 1 10.144927536231885\n",
      "Top 5 42.7536231884058\n",
      "Top 10 59.42028985507246\n",
      "Top 20 80.43478260869566\n"
     ]
    }
   ],
   "source": [
    "tops = [1, 5, 10, 20]\n",
    "for n in tops:\n",
    "    print('Top', n, neighbour_score(concept_dict_f2v, model, top = n))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PLSR 50 Scores\n",
      "Train: 50.57949454864544\n",
      "Test: 40.25185648542887\n"
     ]
    }
   ],
   "source": [
    "print('PLSR 50 Scores')\n",
    "print('Train:', np.mean(feature_score(plsr50, data_type = 'train', max_features = 0))*100)\n",
    "print('Test:', np.mean(feature_score(plsr50, data_type = 'test', max_features = 0))*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PLSR 120 Scores\n",
      "Train: 65.42766605331973\n",
      "Test: 40.71468254150691\n"
     ]
    }
   ],
   "source": [
    "print('PLSR 120 Scores')\n",
    "print('Train:', np.mean(feature_score(plsr120, data_type = 'train', max_features = 0))*100)\n",
    "print('Test:', np.mean(feature_score(plsr120, data_type = 'test', max_features = 0))*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature2Vec Scores\n",
      "Train: 89.12740218635554\n",
      "Test: 44.46290960352884\n"
     ]
    }
   ],
   "source": [
    "print('Feature2Vec Scores')\n",
    "print('Train:', np.mean(feature_score(model, data_type = 'train', max_features = 0))*100)\n",
    "print('Test:', np.mean(feature_score(model, data_type = 'test', max_features = 0))*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}