analysis_topic_modelling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "import pyLDAvis\n",
    "import pandas as pd\n",
    "import pyLDAvis.gensim_models\n",
    "import matplotlib.pyplot as plt\n",
    "from numpy import array\n",
    "from gensim import corpora, models\n",
    "from gensim.models import Phrases\n",
    "from gensim.models.ldamodel import LdaModel\n",
    "from gensim.corpora.dictionary import Dictionary\n",
    "from gensim.models.coherencemodel import CoherenceModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the survey_clean.csv data\n",
    "survey_data = pd.read_csv(\"./data/survey_clean_stemmed.csv\")\n",
    "messages = survey_data[\"stemmed_message\"]\n",
    "messages_list = [i.split() for i in messages]\n",
    "\n",
    "print(len(messages_list))\n",
    "print(messages_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create bigrams and trigrams models\n",
    "bigram = Phrases(messages_list, min_count=10)\n",
    "trigram = Phrases(bigram[messages_list])\n",
    "\n",
    "# Print trigram example\n",
    "print(trigram[bigram[messages_list[0]]])\n",
    "\n",
    "for idx in range(len(messages_list)):\n",
    "    for token in bigram[messages_list[idx]]:\n",
    "        if '_' in token:\n",
    "            # Token is a bigram, add to document.\n",
    "            messages_list[idx].append(token)\n",
    "    for token in trigram[bigram[messages_list[idx]]]:\n",
    "        if '_' in token:\n",
    "            # Token is a bigram, add to document.\n",
    "            messages_list[idx].append(token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a dictionary representation of the documents.\n",
    "dictionary = corpora.Dictionary(messages_list)\n",
    "dictionary.filter_extremes(no_below=5, no_above=0.2)\n",
    "\n",
    "print(dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://radimrehurek.com/gensim/tut1.html\n",
    "# build corpus\n",
    "# Converting list of documents (corpus) into Document Term Matrix using dictionary prepared above.\n",
    "doc_term_matrix = [dictionary.doc2bow(doc) for doc in messages_list]\n",
    "# The function doc2bow converts document (a list of words) into the bag-of-words format\n",
    "\"\"\"The function doc2bow() simply counts the number of occurrences of each distinct word,\n",
    "converts the word to its integer word id and returns the result as a sparse vector.\n",
    "The sparse vector [(0, 1), (1, 1)] therefore reads: in the document “Human computer interaction”,\n",
    "the words computer (id 0) and human (id 1) appear once;\n",
    "the other ten dictionary words appear (implicitly) zero times.\"\"\"\n",
    "print(len(doc_term_matrix))\n",
    "print(doc_term_matrix[100])\n",
    "tfidf = models.TfidfModel(doc_term_matrix)  # build TF-IDF model\n",
    "corpus_tfidf = tfidf[doc_term_matrix]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to compute coherence values\n",
    "def compute_coherence_values(dictionary, corpus, texts, limit, start, step):\n",
    "    coherence_values = []\n",
    "    model_list = []\n",
    "    for num_topics in range(start, limit, step):\n",
    "        model = LdaModel(\n",
    "            corpus=corpus,\n",
    "            id2word=dictionary,\n",
    "            num_topics=num_topics,\n",
    "            iterations=100,\n",
    "            alpha=\"auto\",\n",
    "            per_word_topics=True,\n",
    "        )\n",
    "        model_list.append(model)\n",
    "        coherencemodel = CoherenceModel(\n",
    "            model=model, texts=texts, dictionary=dictionary, coherence=\"c_v\"\n",
    "        )\n",
    "        coherence_values.append(coherencemodel.get_coherence())\n",
    "\n",
    "    return model_list, coherence_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start = 1\n",
    "limit = 21\n",
    "step = 1\n",
    "model_list, coherence_values = compute_coherence_values(\n",
    "    dictionary,\n",
    "    corpus=corpus_tfidf,\n",
    "    texts=messages_list,\n",
    "    start=start,\n",
    "    limit=limit,\n",
    "    step=step,\n",
    ")\n",
    "\n",
    "x = range(start, limit, step)\n",
    "plt.plot(x, coherence_values)\n",
    "plt.xlabel(\"Num Topics\")\n",
    "plt.ylabel(\"Coherence score\")\n",
    "plt.legend((\"coherence_values\"), loc=\"best\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Print the coherence scores\n",
    "for m, cv in zip(x, coherence_values):\n",
    "    print(\"Num Topics =\", m, \" has Coherence Value of\", round(cv, 6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LdaModel(\n",
    "    corpus=corpus_tfidf,\n",
    "    id2word=dictionary,\n",
    "    num_topics=5,\n",
    "    iterations=100,\n",
    "    alpha=\"auto\",\n",
    "    per_word_topics=True,\n",
    ")\n",
    "for idx, topic in model.print_topics(-1):\n",
    "    print(\"Topic: {} Word: {}\".format(idx, topic))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_words_per_topic = []\n",
    "for t in range(model.num_topics):\n",
    "    top_words_per_topic.extend([(t,) + x for x in model.show_topic(t, topn=10)])\n",
    "df = pd.DataFrame(top_words_per_topic, columns=[\"Topic\", \"Word\", \"P\"]).to_csv(\n",
    "    \"./data/top_words2.csv\"\n",
    ")\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pyLDAvis.enable_notebook()\n",
    "data = pyLDAvis.gensim_models.prepare(model, corpus_tfidf, dictionary)\n",
    "print(data)\n",
    "pyLDAvis.save_html(data, \"./data/topic_modelling.html\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}