Quick_Main.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "heUkG4JdNH3j"
   },
   "source": [
    "# Loading the Colab Essentials"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wLdLWDWqmGar"
   },
   "source": [
    "## Mouting the Drive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "tD8sFDHN4SE_",
    "outputId": "56799165-e19f-41a5-969e-c6107c027589"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "#Loading the colab essentials\n",
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "x6dW0OS3jRZw"
   },
   "source": [
    "##Installing some library\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mw0dLLQXs5BT"
   },
   "source": [
    "## Loading the Target Directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "TzXbKOpG7yLe",
    "outputId": "5a56463d-b773-4126-a37e-22d6631db67f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/content/drive/MyDrive/Projects/author clustering\n"
     ]
    }
   ],
   "source": [
    "%cd drive/MyDrive/Projects/author\\ clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "coRWnKApGaGO"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import glob\n",
    "import json\n",
    "import os\n",
    "from sklearn.cluster import  KMeans\n",
    "from sklearn.cluster import AgglomerativeClustering\n",
    "import sortedcollections\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from itertools import combinations\n",
    "import itertools"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ddImI6frNmKU"
   },
   "source": [
    "#Script"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "GyBuwZPwJZH3",
    "outputId": "ea07924a-1a2b-49cf-da49-379468691e9c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  7\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  7\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  7\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  10\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  8\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  8\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  8\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  7\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  7\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  9\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  7\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  9\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  7\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  7\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  7\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  7\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  7\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  7\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  9\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  12\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  7\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  9\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  9\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  8\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  7\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  8\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  8\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  8\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  9\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  7\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  7\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  7\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  10\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  7\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  9\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  9\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  7\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  7\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  8\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  7\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  7\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  7\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  6\n",
      "____________________________________________________________________________________________________\n",
      "____________________________________________________________________________________________________\n",
      "Optimized Cluster Number:  7\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def info(eval_dir):\n",
    "    path=os.path.join(eval_dir,\"info.json\")\n",
    "    s=open(path,'r')\n",
    "    data = json.load(s) \n",
    "    dict_file = {}\n",
    "    for i in range(len(data)):\n",
    "        dict_file[data[i][str(\"folder\")]] = data[i][str(\"language\")]  \n",
    "    return dict_file\n",
    "\n",
    "\n",
    "\n",
    "def clustering_word2vec(data_features,clustering_type='Agglomerative',optimizer_type = 'silhouette'):\n",
    "    d={}\n",
    "    # Finding the scores for cluster number ranging from 6 to Number of documents in a problem folder    \n",
    "    for i in range(6,len(data_features)):\n",
    "        if clustering_type == 'KMeans':\n",
    "          spectral = KMeans(n_clusters=i).fit((np.array(data_features)))\n",
    "        if clustering_type == 'Agglomerative':\n",
    "          spectral = AgglomerativeClustering(n_clusters=i, linkage='ward').fit(np.array(data_features))\n",
    "\n",
    "        label = spectral.fit_predict((np.array(data_features)))\n",
    "\n",
    "        if optimizer_type == 'silhouette':\n",
    "          score= metrics.silhouette_score((np.array(data_features)), label,metric='euclidean')\n",
    "        if optimizer_type == 'calinski':\n",
    "          score=metrics.calinski_harabasz_score((np.array(data_features)), label)\n",
    "\n",
    "        d[i]=score\n",
    "\n",
    "    # Finding the cluster Number with the highest score\n",
    "    n_c=0   \n",
    "    for key,val in d.items():\n",
    "        if(val==max(d.values())):\n",
    "            n_c=key\n",
    "            break\n",
    "\n",
    "    print(\"_\"*100)\n",
    "    print(\"Optimized Cluster Number: \", n_c)\n",
    "    print(\"_\"*100)\n",
    "\n",
    "    # Finally choosing the optimized cluster Number for doing the final clustering \n",
    "    if clustering_type == 'KMeans':\n",
    "      spectral == KMeans(n_clusters=n_c).fit((np.array(data_features)))\n",
    "    if clustering_type == 'Agglomerative':\n",
    "      spectral = AgglomerativeClustering(n_clusters=n_c, linkage='ward').fit(np.array(data_features))\n",
    "\n",
    "    label = spectral.fit_predict((np.array(data_features)))\n",
    "\n",
    "    return label\n",
    "    \n",
    "def prod_output(eval_dir,out_dir,k,labels):\n",
    "    prblm_path=os.path.join(eval_dir,k)\n",
    "    doc_path=glob.glob(prblm_path + '/*')\n",
    "    doc_list_name=[]\n",
    "    for i in doc_path:\n",
    "        m=i.split('/')\n",
    "        m=m[-1]\n",
    "        doc_list_name.append(m)\n",
    "    dic={}\n",
    "    for i,j in zip(doc_list_name,labels):\n",
    "        dic[i]=j \n",
    "        \n",
    "    list_all = []\n",
    "    list_val = []\n",
    "    for v in dic.values():\n",
    "        list_val.append(v)\n",
    "        set_val = set(list_val)\n",
    "    for val in set_val:\n",
    "        list_per_cluster = []\n",
    "        for key, value in dic.items():\n",
    "            if val == value:\n",
    "                list_per_cluster.append(key)\n",
    "        list_all.append(list_per_cluster)\n",
    "     \n",
    "    list_all_output = []\n",
    "    for i in range(len(list_all)):\n",
    "        list_cluster = []\n",
    "        for j in range(len(list_all[i])):\n",
    "            dict_per_doc = {}\n",
    "            dict_per_doc[\"document\"] = list_all[i][j]\n",
    "            list_cluster.append(dict_per_doc)\n",
    "        list_all_output.append(list_cluster)\n",
    "    if(os.path.exists(out_dir+\"/\"+k)==False):    \n",
    "        os.mkdir(out_dir+\"/\"+k)    \n",
    "    \n",
    "    out_folder=out_dir + '/' +k\n",
    "    out_path = out_folder + \"/clustering.json\"\n",
    "    out_file = open(out_path, \"w\")\n",
    "\n",
    "    json.dump(list_all_output, out_file, indent=4)\n",
    "    return list_all_output   \n",
    "\n",
    "def similarity_score(list_all, dict_features):\n",
    "    list_all_comb = []\n",
    "    for i in range(len(list_all)):\n",
    "        list_comb_percluster = []\n",
    "        if len(list_all[i]) > 1:\n",
    "            for j in range(len(list_all[i])):\n",
    "                list_comb_percluster.append(list_all[i][j][\"document\"])\n",
    "            list_all_comb.append(list_comb_percluster)\n",
    "    combs = []\n",
    "    for i in range(len(list_all_comb)):\n",
    "        comb = list(combinations(list_all_comb[i], 2))\n",
    "        combs.append(comb)\n",
    "    comb_list = list(itertools.chain(*combs))\n",
    "    \n",
    "    all_sim = []\n",
    "    for i in range(len(comb_list)):\n",
    "        doc1 = comb_list[i][0].split(\",\")\n",
    "        doc2 = comb_list[i][1].split(\",\")\n",
    "        vec1 = dict_features[doc1[0]]\n",
    "        vec2 = dict_features[doc2[0]]\n",
    "        vec1=[vec1]\n",
    "        vec2=[vec2]\n",
    "        sim = cosine_similarity(vec1, vec2)\n",
    "        all_sim.append(sim)\n",
    "    return comb_list, all_sim\n",
    "\n",
    "def write_ranking(comb_list, all_sim, out_dir,k):\n",
    "    list_all_output = []\n",
    "    for i in range(len(comb_list)):\n",
    "        dict_sim_perpair = {}\n",
    "        dict_sim_perpair[\"document1\"] = comb_list[i][0]\n",
    "        dict_sim_perpair[\"document2\"] = comb_list[i][1]\n",
    "        dict_sim_perpair[\"score\"] = round(all_sim[i][0][0],6)\n",
    "        list_all_output.append(dict_sim_perpair)\n",
    "    out_folder=out_dir + '/' +k\n",
    "    out_path = out_folder + \"/ranking.json\"\n",
    "    out_file = open(out_path, \"w\")\n",
    "\n",
    "    json.dump(list_all_output, out_file, indent=4)\n",
    "    return list_all_output    \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "if __name__==\"__main__\":\n",
    "    eval_dir=\"./quick_run_main/pan17-author-clustering-test-dataset-2017-03-14\"\n",
    "    out_dir=\"./quick_run_main/tfidf_weight_x_word2vec_output_pan17-author-clustering-test-dataset-2017-03-14\"\n",
    "\n",
    "    dict_f=info(eval_dir)  \n",
    "    dict_f=sortedcollections.OrderedDict(sorted(dict_f.items()))\n",
    "\n",
    "    en_=open(\"./quick_run_main/english_weighted.json\",'r')\n",
    "    dt_=open(\"./quick_run_main/dutch_weighted.json\",'r')\n",
    "    gr_=open(\"./quick_run_main/greek_weighted.json\",'r')\n",
    "\n",
    "    vec_en=json.load(en_)\n",
    "    vec_dt=json.load(dt_)\n",
    "    vec_gr=json.load(gr_)\n",
    "\n",
    "    for k,v in dict_f.items():\n",
    "        \n",
    "        if v==\"en\":\n",
    "            vectors=vec_en[k]\n",
    "            labels=clustering_word2vec(vectors)\n",
    "           \n",
    "        \n",
    "            \n",
    "        \n",
    "        elif v==\"nl\":\n",
    "            vectors=vec_dt[k]\n",
    "            labels=clustering_word2vec(vectors)\n",
    "            \n",
    "            \n",
    "        elif v==\"gr\":\n",
    "            vectors=vec_gr[k]\n",
    "            labels=clustering_word2vec(vectors)\n",
    "            \n",
    "        list_all=prod_output(eval_dir,out_dir,k,labels)  \n",
    "\n",
    "        dict_features={}\n",
    "\n",
    "        prblm_path=os.path.join(eval_dir,k)\n",
    "        doc_path=glob.glob(prblm_path + '/*')\n",
    "        doc_list_name=[]\n",
    "        for i in doc_path:\n",
    "            m=i.split('/')\n",
    "            m=m[-1]\n",
    "            doc_list_name.append(m)\n",
    "        i=0    \n",
    "        for j in doc_list_name:\n",
    "            dict_features[j]=vectors[i]\n",
    "            i=i+1\n",
    "        # similarity between documents\n",
    "        list_comb, all_sim = similarity_score(list_all, dict_features)\n",
    "        list_sim = write_ranking(list_comb, all_sim, out_dir,k)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lyLV10uwxvd5"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "name": "author.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}