DaDoEval_2020.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import csv\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from skopt import BayesSearchCV\n",
    "from skopt.space import Real\n",
    "\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.datasets import load_files\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.preprocessing import FunctionTransformer, MinMaxScaler\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold\n",
    "from sklearn.metrics import f1_score, precision_score, recall_score, accuracy_score, plot_confusion_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading training and test sets as sklearn bunch objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training-set\n",
    "train_dataset = load_files(\"CoarseTask/training_set/\", encoding=\"utf-8\")\n",
    "\n",
    "# cross-genre test-set\n",
    "test_dataset_cross = load_files(\"CoarseTask/test_cross\", encoding=\"utf-8\")\n",
    "\n",
    "# same-genre test-set\n",
    "test_dataset_same = load_files(\"CoarseTask/test_same\", encoding=\"utf-8\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining the feature space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_document_length(dataset):\n",
    "    \"\"\" Splits each document resorting to a naive white-space-split strategy\n",
    "    and counts the number of word tokens.\n",
    "    \"\"\"\n",
    "    return np.array([len(doc.split(\" \")) for doc in dataset]).reshape(-1, 1)\n",
    "\n",
    "# combines n-grams and document length\n",
    "combined_features = FeatureUnion([\n",
    "    (\"tfidf_word\", TfidfVectorizer(analyzer='word', ngram_range=(1, 2), lowercase=False)),\n",
    "    (\"tfidf_char\", TfidfVectorizer(analyzer='char', ngram_range=(3, 5), lowercase=False)),\n",
    "    (\"document_len\", Pipeline([\n",
    "        (\"transf\", FunctionTransformer(get_document_length, validate=False)),\n",
    "        (\"scaler\", MinMaxScaler())\n",
    "    ]))\n",
    "], n_jobs=-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tuning hyper-parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_val, y_train, y_val = train_test_split(train_dataset.data, train_dataset.target, test_size=0.2, random_state=42)\n",
    "\n",
    "pipeline = Pipeline([\n",
    "    (\"features\", combined_features),\n",
    "    (\"clf\", SVC(kernel='linear', decision_function_shape='ovo', tol=1e-12, class_weight='balanced', random_state=42))\n",
    "])\n",
    "\n",
    "# hyper-parameters search space\n",
    "parameters = {\n",
    "    \"features__tfidf_word__max_df\": Real(.3, .9, prior='log-uniform'),\n",
    "    \"features__tfidf_word__min_df\": Real(.001, .1, prior='log-uniform'),\n",
    "    \n",
    "    \"features__tfidf_char__max_df\": Real(.3, .9, prior='log-uniform'),\n",
    "    \"features__tfidf_char__min_df\": Real(.001, .1, prior='log-uniform'),\n",
    "        \n",
    "    \"clf__C\": Real(1e-6, 100.0, prior='log-uniform')\n",
    "}\n",
    "\n",
    "stratified_kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "bayes_search = BayesSearchCV(pipeline, parameters, cv=stratified_kf, n_iter=30, n_jobs=-1, verbose=1, scoring=\"f1_macro\")\n",
    "bayes_search.fit(X_train, y_train)\n",
    "\n",
    "# prints optimized hyper-parameters\n",
    "best_parameters = bayes_search.best_estimator_.get_params()\n",
    "for param_name in sorted(parameters.keys()):\n",
    "    print('{:30} {}'.format(param_name, round(best_parameters[param_name], 3)))\n",
    "\n",
    "# predicts labels for the validation set\n",
    "tuned_clf = bayes_search.best_estimator_\n",
    "val_prediction = tuned_clf.predict(X_val)\n",
    "\n",
    "# prints relevant statistics and plots the confusion matrix\n",
    "print('\\n{:30}{}'.format('accuracy:', round((accuracy_score(y_val, val_prediction)), 2)))\n",
    "print('{:30}{}'.format('macro-averaged precision:', round(precision_score(y_val, val_prediction, average='macro'), 2)))\n",
    "print('{:30}{}'.format('macro-averaged recall:', round(recall_score(y_val, val_prediction, average='macro'), 2)))\n",
    "print('{:30}{}'.format('macro-averaged f1 score:', round(f1_score(y_val, val_prediction, average='macro'), 2)))\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(10, 10))\n",
    "labels = [\"1901-1918\", \"1919-1926\", \"1927-1942\", \"1943-1947\", \"1948-1954\"]\n",
    "plot_confusion_matrix(tuned_clf, X_val, y_val, display_labels=labels, cmap=plt.cm.Blues, ax=ax)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predicting labels for same-genre and cross-genre test-sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = train_dataset.data\n",
    "y_train = train_dataset.target\n",
    "\n",
    "# same-genre\n",
    "X_test_same_genre = test_dataset_same.data\n",
    "y_test_same_genre = test_dataset_same.target\n",
    "\n",
    "# cross-genre\n",
    "X_test_cross_genre = test_dataset_cross.data\n",
    "y_test_cross_genre = test_dataset_cross.target\n",
    "\n",
    "# re-train tuned classifier on whole training set\n",
    "tuned_clf.fit(X_train, y_train)\n",
    "\n",
    "predicted_cross = tuned_clf.predict(X_test_cross_genre)\n",
    "predicted_same = tuned_clf.predict(X_test_same_genre)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Storing predictions as tsv files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('2_matteo-brv_2.tsv', 'wt') as out_file:\n",
    "    tsv_writer = csv.writer(out_file, delimiter='\\t')\n",
    "    for name, pred in zip(test_dataset_cross.filenames, predicted_cross):\n",
    "        name = re.sub(r'.*/', '', name)\n",
    "        tsv_writer.writerow([name, f'Class{pred+1}'])\n",
    "\n",
    "with open('1_matteo-brv_2.tsv', 'wt') as out_file:\n",
    "    tsv_writer = csv.writer(out_file, delimiter='\\t')\n",
    "    for name, pred in zip(test_dataset_same.filenames, predicted_same):\n",
    "        name = re.sub(r'.*/', '', name)\n",
    "        tsv_writer.writerow([name, f'Class{pred+1}'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}