Text_comparison_and_similarity_measurement.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "38tIxoAzZmEA"
      },
      "outputs": [],
      "source": [
        "pip install sentence-transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hX2B_0klaECH"
      },
      "outputs": [],
      "source": [
        "from sentence_transformers import SentenceTransformer, util\n",
        "import pandas as pd\n",
        "import re\n",
        "import networkx as nx\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "from flask import Flask, render_template_string\n",
        "import matplotlib.pyplot as plt\n",
        "import io\n",
        "import base64\n",
        "from torch import embedding\n",
        "!pip install flask-ngrok\n",
        "from flask import Flask\n",
        "from flask_ngrok import run_with_ngrok\n",
        "from flask import Flask, request\n",
        "import matplotlib.pyplot as plt\n",
        "from scipy.cluster.hierarchy import linkage, dendrogram, fcluster\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "stCXRKHRaKuH"
      },
      "outputs": [],
      "source": [
        "!curl -O https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip\n",
        "!unzip -o ngrok-stable-linux-amd64.zip\n",
        "!chmod +x ./ngrok\n",
        "!./ngrok authtoken 2YzojbvlvYkzVrjmqyym1IkjxbN_rs4jcvbEkQ7Nfi5jbCiD\n",
        "\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6aMXDyfVaPpH"
      },
      "outputs": [],
      "source": [
        "\n",
        "\n",
        "  app = Flask(__name__)\n",
        "  run_with_ngrok(app)  # Start ngrok when the app is run\n",
        "\n",
        "  @app.route('/')\n",
        "  def display_output():\n",
        "\n",
        "\n",
        "\n",
        "      # Load the dataset\n",
        "      df = pd.read_csv('/content/drive/MyDrive/Colab Notebooks/collection_jobs.csv', delimiter='|', error_bad_lines=False)\n",
        "\n",
        "      # Preprocess and clean text data\n",
        "      df['title'] = df['title'].str.lower()  # Lowercase\n",
        "      df['description'] = df['description'].str.lower()  # Lowercase\n",
        "\n",
        "      # Remove rows with all whitespace in all columns\n",
        "      df = df.dropna(how='all', subset=df.columns[1:], inplace=False)\n",
        "\n",
        "      # Define a regular expression pattern to match timestamps\n",
        "      timestamp_pattern = r'\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}'\n",
        "\n",
        "      df = df[~df['description'].str.contains(timestamp_pattern, na=False)]\n",
        "\n",
        "      # Reset the DataFrame index\n",
        "      df.reset_index(drop=True, inplace=True)\n",
        "\n",
        "      columns_to_remove = ['id', 'raw_jobs_crawl_id', 'title', 'employer_description', 'employer_description', 'insertion_date', 'occupation']\n",
        "      df = df.drop(columns=columns_to_remove)\n",
        "\n",
        "      # Load the pre-trained model\n",
        "      model = SentenceTransformer('flax-sentence-embeddings/all_datasets_v3_MiniLM-L6')\n",
        "\n",
        "      # Choose a subset of rows to compare the job descriptions\n",
        "      values= 50;\n",
        "      #values= 100;\n",
        "      #values= 500;\n",
        "      subset_df = df.head(values)\n",
        "      sentences = subset_df['description'].tolist()\n",
        "\n",
        "      # Generate HTML content for the table\n",
        "      table_html = f'''\n",
        "      <div id=\"table-tab\" class=\"tab-content\">\n",
        "        <h2>Job Descriptions </h2>\n",
        "        <table style=\"border-collapse: collapse; width: 100%; border: 1px solid #ddd;\">\n",
        "            <thead>\n",
        "                <tr style=\"border: 1px solid #ddd; background-color: #f2f2f2;\">\n",
        "                <th style=\"padding: 8px; text-align: left; border: 1px solid #ddd;\">Description Number</th>\n",
        "                <th style=\"padding: 8px; text-align: left; border: 1px solid #ddd;\">Description Details</th>\n",
        "            </tr>\n",
        "            </thead>\n",
        "            <tbody>\n",
        "    '''\n",
        "\n",
        "      # Python code to insert descriptions into the HTML\n",
        "      for index, description in enumerate(sentences):\n",
        "          table_html += '''\n",
        "                <tr style=\"border: 1px solid #ddd;\">\n",
        "                    <td style=\"padding: 8px; text-align: left; border: 1px solid #ddd;\">Description {}</td>\n",
        "                    <td style=\"padding: 8px; text-align: left; border: 1px solid #ddd;\">{}</td>\n",
        "                </tr>\n",
        "        '''.format(index + 1, description)\n",
        "\n",
        "      # Encode the sentences\n",
        "      embeddings = model.encode(sentences)\n",
        "\n",
        "      # Compute cosine similarity between all pairs\n",
        "      cos_sim = util.cos_sim(embeddings, embeddings)\n",
        "\n",
        "      # Create a graph\n",
        "      G = nx.Graph()\n",
        "      for i, _ in enumerate(sentences):\n",
        "          G.add_node(i, label=f\"Description {i+1}\")\n",
        "      for i in range(len(sentences)):\n",
        "          for j in range(i + 1, len(sentences)):\n",
        "              similarity = cos_sim[i][j]\n",
        "              G.add_edge(i, j, weight=similarity)\n",
        "\n",
        "\n",
        "      fig, ax2 = plt.subplots(1, figsize=(values, values))\n",
        "\n",
        "      annot_font = {'size': 10, 'weight': 'bold'}\n",
        "      # Plot the cosine similarity matrix\n",
        "      ax2.set_title(\"Similarity Matrix For Jobs\")\n",
        "      sns.heatmap(cos_sim, annot=True, xticklabels=[f\"Description {i+1}\" for i in range(len(sentences))],yticklabels=[f\"Description {i+1}\" for i in range(len(sentences))], cmap=\"YlGnBu\", ax=ax2, annot_kws=annot_font)\n",
        "\n",
        "      # Perform hierarchical clustering\n",
        "      linkage_matrix = linkage(cos_sim, method='ward')\n",
        "\n",
        "      # Save plot to a buffer\n",
        "      buffer = io.BytesIO()\n",
        "      plt.savefig(buffer, format='png')\n",
        "      buffer.seek(0)\n",
        "\n",
        "      # Encode plot to base64 for HTML display\n",
        "      plot_data = base64.b64encode(buffer.getvalue()).decode('utf-8')\n",
        "      plt.close()\n",
        "\n",
        "      javascript_code = '''\n",
        "<script>\n",
        "function openTab(evt, tabName) {\n",
        "        var i, tabcontent, tablinks;\n",
        "        tabcontent = document.getElementsByClassName(\"tab-content\");\n",
        "        for (i = 0; i < tabcontent.length; i++) {\n",
        "            tabcontent[i].style.display = \"none\";\n",
        "        }\n",
        "        tablinks = document.getElementsByTagName(\"li\");\n",
        "        for (i = 0; i < tablinks.length; i++) {\n",
        "            tablinks[i].className = \"\";\n",
        "        }\n",
        "        document.getElementById(tabName).style.display = \"block\";\n",
        "        evt.currentTarget.className = \"active\";\n",
        "    }\n",
        "\n",
        "    function zoomIn() {\n",
        "        var img = document.getElementById('graph-image');\n",
        "        var currWidth = img.clientWidth;\n",
        "        img.style.width = (currWidth + 100) + 'px';\n",
        "    }\n",
        "\n",
        "    function zoomOut() {\n",
        "        var img = document.getElementById('graph-image');\n",
        "        var currWidth = img.clientWidth;\n",
        "        if (currWidth > 100) {\n",
        "            img.style.width = (currWidth - 100) + 'px';\n",
        "        }\n",
        "    }\n",
        "\n",
        "</script>\n",
        "'''\n",
        "\n",
        "\n",
        "      # HTML to display the plot, matrix heatmap, and search results in the browser\n",
        "      html_content = f'''\n",
        "      <!DOCTYPE html>\n",
        "      <html>\n",
        "      <html>\n",
        "  <head>\n",
        "      <title>Text Similarity Visualization</title>\n",
        "      <style>\n",
        "        /* Your CSS styles for tabs and content */\n",
        "        body {{\n",
        "            font-family: Arial, sans-serif;\n",
        "            margin: 20px;\n",
        "            background-color: #f4f4f4;\n",
        "        }}\n",
        "        .tabs ul {{\n",
        "            list-style-type: none;\n",
        "            padding: 0;\n",
        "            margin: 0;\n",
        "        }}\n",
        "        .tabs ul li {{\n",
        "            display: inline-block;\n",
        "            margin-right: 10px;\n",
        "            cursor: pointer;\n",
        "            padding: 10px;\n",
        "            background-color: #ddd;\n",
        "            border-radius: 5px 5px 0 0;\n",
        "        }}\n",
        "        .tabs ul li.active {{\n",
        "            font-weight: bold;\n",
        "            background-color: #f4f4f4;\n",
        "        }}\n",
        "        .tab-content {{\n",
        "            display: none;\n",
        "            padding: 20px;\n",
        "            border: 1px solid #ccc;\n",
        "            border-top: none;\n",
        "            background-color: #fff;\n",
        "            border-radius: 0 0 5px 5px;\n",
        "        }}\n",
        "        .tab-content.active {{\n",
        "            display: block;\n",
        "\n",
        "\n",
        "        /* Additional styles for graph display */\n",
        "        .graph-tab-content {{\n",
        "            text-align: center;\n",
        "        }}\n",
        "        .graph-tab-content img {{\n",
        "            max-width: 80%;\n",
        "            display: block;\n",
        "            margin: 0 auto;\n",
        "        }}\n",
        "        .graph-tab-content button {{\n",
        "            margin-top: 10px;\n",
        "            padding: 8px 20px;\n",
        "            border-radius: 5px;\n",
        "            background-color: #007bff;\n",
        "            color: white;\n",
        "            border: none;\n",
        "            cursor: pointer;\n",
        "        }}\n",
        "        .graph-tab-content button:hover {{\n",
        "            background-color: #0056b3;\n",
        "        }}\n",
        "\n",
        "\n",
        "    </style>\n",
        "  </head>\n",
        "      <body>\n",
        "      <div class=\"tabs\">\n",
        "    <ul>\n",
        "        <li onclick=\"openTab(event, 'welcome-tab')\" class=\"active\">Welcome</li>\n",
        "        <li onclick=\"openTab(event, 'graph-tab')\">Graph</li>\n",
        "        <li onclick=\"openTab(event, 'table-tab')\">Table</li>\n",
        "\n",
        "    </ul>\n",
        "</div>\n",
        "<div id=\"welcome-tab\" class=\"tab-content active\">\n",
        "\n",
        "    <h2>We welcome you to our project </h2>\n",
        "\n",
        "</div>\n",
        "\n",
        "<div id=\"graph-tab\" class=\"tab-content\">\n",
        "    <h2>Sentence Similarity Graph</h2>\n",
        "    <img id=\"graph-image\" src=\"data:image/png;base64, {plot_data}\" alt=\"Cosine Similarity Graph\"  style=\"float: left; margin-right: 20px;\"/>\n",
        "\n",
        "    <!-- Your graph content here -->\n",
        "    <!-- Zoom buttons and JavaScript code -->\n",
        "    <button onclick=\"zoomIn()\" style=\"background-color: #4CAF50; /* Green */\n",
        "  border: none;\n",
        "  color: white;\n",
        "  padding: 15px 32px;\n",
        "  text-align: center;\n",
        "  text-decoration: none;\n",
        "  display: inline-block;\n",
        "  font-size: 16px;\n",
        "  margin: 4px 2px;\n",
        "  cursor: pointer;\">Zoom In</button>\n",
        "\n",
        "<button onclick=\"zoomOut()\" style=\"background-color: #008CBA; /* Blue */\n",
        "  border: none;\n",
        "  color: white;\n",
        "  padding: 15px 32px;\n",
        "  text-align: center;\n",
        "  text-decoration: none;\n",
        "  display: inline-block;\n",
        "  font-size: 16px;\n",
        "  margin: 4px 2px;\n",
        "  cursor: pointer;\">Zoom Out</button>\n",
        "\n",
        "    {javascript_code}\n",
        "</div>\n",
        "\n",
        "\n",
        "\n",
        "{table_html}\n",
        "\n",
        "      </body>\n",
        "      </html>\n",
        "      '''\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "      return html_content\n",
        "\n",
        "  if __name__ == '__main__':\n",
        "      app.run()\n"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}