data_cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "72879b89-4a16-46c7-a336-863b2d548c32",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd   # importing library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "25e0e324-755a-4930-9553-801977691e6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('dataset/DataScientist.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "797d78d6-0dbd-463c-a896-78e798a268d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>index</th>\n",
       "      <th>Job Title</th>\n",
       "      <th>Salary Estimate</th>\n",
       "      <th>Job Description</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Company Name</th>\n",
       "      <th>Location</th>\n",
       "      <th>Headquarters</th>\n",
       "      <th>Size</th>\n",
       "      <th>Founded</th>\n",
       "      <th>Type of ownership</th>\n",
       "      <th>Industry</th>\n",
       "      <th>Sector</th>\n",
       "      <th>Revenue</th>\n",
       "      <th>Competitors</th>\n",
       "      <th>Easy Apply</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Senior Data Scientist</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>ABOUT HOPPER\\n\\nAt Hopper, we’re on a mission ...</td>\n",
       "      <td>3.5</td>\n",
       "      <td>Hopper\\n3.5</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>Montreal, Canada</td>\n",
       "      <td>501 to 1000 employees</td>\n",
       "      <td>2007</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Travel Agencies</td>\n",
       "      <td>Travel &amp; Tourism</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Data Scientist, Product Analytics</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>At Noom, we use scientifically proven methods ...</td>\n",
       "      <td>4.5</td>\n",
       "      <td>Noom US\\n4.5</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>1001 to 5000 employees</td>\n",
       "      <td>2008</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Health, Beauty, &amp; Fitness</td>\n",
       "      <td>Consumer Services</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>Data Science Manager</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>Decode_M\\n\\nhttps://www.decode-m.com/\\n\\nData ...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>Decode_M</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>1 to 50 employees</td>\n",
       "      <td>-1</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>Data Analyst</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>Sapphire Digital seeks a dynamic and driven mi...</td>\n",
       "      <td>3.4</td>\n",
       "      <td>Sapphire Digital\\n3.4</td>\n",
       "      <td>Lyndhurst, NJ</td>\n",
       "      <td>Lyndhurst, NJ</td>\n",
       "      <td>201 to 500 employees</td>\n",
       "      <td>2019</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Internet</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>Zocdoc, Healthgrades</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>Director, Data Science</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>Director, Data Science - (200537)\\nDescription...</td>\n",
       "      <td>3.4</td>\n",
       "      <td>United Entertainment Group\\n3.4</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>51 to 200 employees</td>\n",
       "      <td>2007</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Advertising &amp; Marketing</td>\n",
       "      <td>Business Services</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>BBDO, Grey Group, Droga5</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>Job Brief\\n\\nThe ideal candidate will have pre...</td>\n",
       "      <td>2.9</td>\n",
       "      <td>IFG Companies\\n2.9</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>Hartford, CT</td>\n",
       "      <td>201 to 500 employees</td>\n",
       "      <td>1985</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Insurance Carriers</td>\n",
       "      <td>Insurance</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>Colony Specialty, Markel, RLI</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>Quantitative Researcher</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>Experience: Entry-level (PhD Program) or Exper...</td>\n",
       "      <td>4.4</td>\n",
       "      <td>PDT Partners\\n4.4</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>51 to 200 employees</td>\n",
       "      <td>1993</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Investment Banking &amp; Asset Management</td>\n",
       "      <td>Finance</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>Quantitative Research Associate</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>Seeking a quant to work with senior researcher...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>Enlightenment Research</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>1 to 50 employees</td>\n",
       "      <td>-1</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>AI Scientist</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>Paige is a software company helping pathologis...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Paige\\n5.0</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>1 to 50 employees</td>\n",
       "      <td>2018</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Enterprise Software &amp; Network Solutions</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>Quantitative Researcher</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>About the Position\\n\\n\\nAt Jane Street, we con...</td>\n",
       "      <td>4.8</td>\n",
       "      <td>Jane Street\\n4.8</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>501 to 1000 employees</td>\n",
       "      <td>2000</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Investment Banking &amp; Asset Management</td>\n",
       "      <td>Finance</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  index                          Job Title  \\\n",
       "0           0      0              Senior Data Scientist   \n",
       "1           1      1  Data Scientist, Product Analytics   \n",
       "2           2      2               Data Science Manager   \n",
       "3           3      3                       Data Analyst   \n",
       "4           4      4             Director, Data Science   \n",
       "5           5      5                     Data Scientist   \n",
       "6           6      6            Quantitative Researcher   \n",
       "7           7      7    Quantitative Research Associate   \n",
       "8           8      8                       AI Scientist   \n",
       "9           9      9            Quantitative Researcher   \n",
       "\n",
       "                Salary Estimate  \\\n",
       "0  $111K-$181K (Glassdoor est.)   \n",
       "1  $111K-$181K (Glassdoor est.)   \n",
       "2  $111K-$181K (Glassdoor est.)   \n",
       "3  $111K-$181K (Glassdoor est.)   \n",
       "4  $111K-$181K (Glassdoor est.)   \n",
       "5  $111K-$181K (Glassdoor est.)   \n",
       "6  $111K-$181K (Glassdoor est.)   \n",
       "7  $111K-$181K (Glassdoor est.)   \n",
       "8  $111K-$181K (Glassdoor est.)   \n",
       "9  $111K-$181K (Glassdoor est.)   \n",
       "\n",
       "                                     Job Description  Rating  \\\n",
       "0  ABOUT HOPPER\\n\\nAt Hopper, we’re on a mission ...     3.5   \n",
       "1  At Noom, we use scientifically proven methods ...     4.5   \n",
       "2  Decode_M\\n\\nhttps://www.decode-m.com/\\n\\nData ...    -1.0   \n",
       "3  Sapphire Digital seeks a dynamic and driven mi...     3.4   \n",
       "4  Director, Data Science - (200537)\\nDescription...     3.4   \n",
       "5  Job Brief\\n\\nThe ideal candidate will have pre...     2.9   \n",
       "6  Experience: Entry-level (PhD Program) or Exper...     4.4   \n",
       "7  Seeking a quant to work with senior researcher...    -1.0   \n",
       "8  Paige is a software company helping pathologis...     5.0   \n",
       "9  About the Position\\n\\n\\nAt Jane Street, we con...     4.8   \n",
       "\n",
       "                      Company Name       Location      Headquarters  \\\n",
       "0                      Hopper\\n3.5   New York, NY  Montreal, Canada   \n",
       "1                     Noom US\\n4.5   New York, NY      New York, NY   \n",
       "2                         Decode_M   New York, NY      New York, NY   \n",
       "3            Sapphire Digital\\n3.4  Lyndhurst, NJ     Lyndhurst, NJ   \n",
       "4  United Entertainment Group\\n3.4   New York, NY      New York, NY   \n",
       "5               IFG Companies\\n2.9   New York, NY      Hartford, CT   \n",
       "6                PDT Partners\\n4.4   New York, NY      New York, NY   \n",
       "7           Enlightenment Research   New York, NY      New York, NY   \n",
       "8                       Paige\\n5.0   New York, NY      New York, NY   \n",
       "9                 Jane Street\\n4.8   New York, NY      New York, NY   \n",
       "\n",
       "                     Size  Founded  Type of ownership  \\\n",
       "0   501 to 1000 employees     2007  Company - Private   \n",
       "1  1001 to 5000 employees     2008  Company - Private   \n",
       "2       1 to 50 employees       -1            Unknown   \n",
       "3    201 to 500 employees     2019  Company - Private   \n",
       "4     51 to 200 employees     2007  Company - Private   \n",
       "5    201 to 500 employees     1985  Company - Private   \n",
       "6     51 to 200 employees     1993  Company - Private   \n",
       "7       1 to 50 employees       -1            Unknown   \n",
       "8       1 to 50 employees     2018  Company - Private   \n",
       "9   501 to 1000 employees     2000  Company - Private   \n",
       "\n",
       "                                  Industry                  Sector  \\\n",
       "0                          Travel Agencies        Travel & Tourism   \n",
       "1                Health, Beauty, & Fitness       Consumer Services   \n",
       "2                                       -1                      -1   \n",
       "3                                 Internet  Information Technology   \n",
       "4                  Advertising & Marketing       Business Services   \n",
       "5                       Insurance Carriers               Insurance   \n",
       "6    Investment Banking & Asset Management                 Finance   \n",
       "7                                       -1                      -1   \n",
       "8  Enterprise Software & Network Solutions  Information Technology   \n",
       "9    Investment Banking & Asset Management                 Finance   \n",
       "\n",
       "                    Revenue                    Competitors Easy Apply  \n",
       "0  Unknown / Non-Applicable                             -1         -1  \n",
       "1  Unknown / Non-Applicable                             -1         -1  \n",
       "2  Unknown / Non-Applicable                             -1       True  \n",
       "3  Unknown / Non-Applicable           Zocdoc, Healthgrades         -1  \n",
       "4  Unknown / Non-Applicable       BBDO, Grey Group, Droga5         -1  \n",
       "5  Unknown / Non-Applicable  Colony Specialty, Markel, RLI         -1  \n",
       "6  Unknown / Non-Applicable                             -1         -1  \n",
       "7  Unknown / Non-Applicable                             -1       True  \n",
       "8  Unknown / Non-Applicable                             -1       True  \n",
       "9  Unknown / Non-Applicable                             -1         -1  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "88d0ee31-159f-470b-bc04-6e226552cddb",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.set_index('index', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "901a9b88-c886-42c0-be0b-494614023150",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Job Title</th>\n",
       "      <th>Salary Estimate</th>\n",
       "      <th>Job Description</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Company Name</th>\n",
       "      <th>Location</th>\n",
       "      <th>Headquarters</th>\n",
       "      <th>Size</th>\n",
       "      <th>Founded</th>\n",
       "      <th>Type of ownership</th>\n",
       "      <th>Industry</th>\n",
       "      <th>Sector</th>\n",
       "      <th>Revenue</th>\n",
       "      <th>Competitors</th>\n",
       "      <th>Easy Apply</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Senior Data Scientist</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>ABOUT HOPPER\\n\\nAt Hopper, we’re on a mission ...</td>\n",
       "      <td>3.5</td>\n",
       "      <td>Hopper\\n3.5</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>Montreal, Canada</td>\n",
       "      <td>501 to 1000 employees</td>\n",
       "      <td>2007</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Travel Agencies</td>\n",
       "      <td>Travel &amp; Tourism</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Data Scientist, Product Analytics</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>At Noom, we use scientifically proven methods ...</td>\n",
       "      <td>4.5</td>\n",
       "      <td>Noom US\\n4.5</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>1001 to 5000 employees</td>\n",
       "      <td>2008</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Health, Beauty, &amp; Fitness</td>\n",
       "      <td>Consumer Services</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Data Science Manager</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>Decode_M\\n\\nhttps://www.decode-m.com/\\n\\nData ...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>Decode_M</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>1 to 50 employees</td>\n",
       "      <td>-1</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Data Analyst</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>Sapphire Digital seeks a dynamic and driven mi...</td>\n",
       "      <td>3.4</td>\n",
       "      <td>Sapphire Digital\\n3.4</td>\n",
       "      <td>Lyndhurst, NJ</td>\n",
       "      <td>Lyndhurst, NJ</td>\n",
       "      <td>201 to 500 employees</td>\n",
       "      <td>2019</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Internet</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>Zocdoc, Healthgrades</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Director, Data Science</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>Director, Data Science - (200537)\\nDescription...</td>\n",
       "      <td>3.4</td>\n",
       "      <td>United Entertainment Group\\n3.4</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>51 to 200 employees</td>\n",
       "      <td>2007</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Advertising &amp; Marketing</td>\n",
       "      <td>Business Services</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>BBDO, Grey Group, Droga5</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4375</th>\n",
       "      <td>3904</td>\n",
       "      <td>AWS Data Engineer</td>\n",
       "      <td>$55K-$112K (Glassdoor est.)</td>\n",
       "      <td>About Us\\n\\nTachyon Technologies is a Digital ...</td>\n",
       "      <td>4.4</td>\n",
       "      <td>Tachyon Technologies\\n4.4</td>\n",
       "      <td>Dublin, OH</td>\n",
       "      <td>Irving, TX</td>\n",
       "      <td>201 to 500 employees</td>\n",
       "      <td>2011</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>IT Services</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>$10 to $25 million (USD)</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4376</th>\n",
       "      <td>3905</td>\n",
       "      <td>Data Analyst â Junior</td>\n",
       "      <td>$55K-$112K (Glassdoor est.)</td>\n",
       "      <td>Job description\\nInterpret data, analyze resul...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Staffigo Technical Services, LLC\\n5.0</td>\n",
       "      <td>Columbus, OH</td>\n",
       "      <td>Woodridge, IL</td>\n",
       "      <td>51 to 200 employees</td>\n",
       "      <td>2008</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>IT Services</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>$50 to $100 million (USD)</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4377</th>\n",
       "      <td>3906</td>\n",
       "      <td>Security Analytics Data Engineer</td>\n",
       "      <td>$55K-$112K (Glassdoor est.)</td>\n",
       "      <td>Job DescriptionThe Security Analytics Data Eng...</td>\n",
       "      <td>3.8</td>\n",
       "      <td>PDS Tech, Inc.\\n3.8</td>\n",
       "      <td>Dublin, OH</td>\n",
       "      <td>Irving, TX</td>\n",
       "      <td>5001 to 10000 employees</td>\n",
       "      <td>1977</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Staffing &amp; Outsourcing</td>\n",
       "      <td>Business Services</td>\n",
       "      <td>$100 to $500 million (USD)</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4378</th>\n",
       "      <td>3907</td>\n",
       "      <td>Security Analytics Data Engineer</td>\n",
       "      <td>$55K-$112K (Glassdoor est.)</td>\n",
       "      <td>The Security Analytics Data Engineer will inte...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Data Resource Technologies\\n4.0</td>\n",
       "      <td>Dublin, OH</td>\n",
       "      <td>Omaha, NE</td>\n",
       "      <td>1 to 50 employees</td>\n",
       "      <td>-1</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Accounting</td>\n",
       "      <td>Accounting &amp; Legal</td>\n",
       "      <td>Less than $1 million (USD)</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4379</th>\n",
       "      <td>3908</td>\n",
       "      <td>Patient Safety Physician or Safety Scientist -...</td>\n",
       "      <td>$55K-$112K (Glassdoor est.)</td>\n",
       "      <td>Help us transform patients' lives.\\nAt UCB, we...</td>\n",
       "      <td>3.7</td>\n",
       "      <td>UCB\\n3.7</td>\n",
       "      <td>Slough, OH</td>\n",
       "      <td>Brussel, Belgium</td>\n",
       "      <td>5001 to 10000 employees</td>\n",
       "      <td>-1</td>\n",
       "      <td>Company - Public</td>\n",
       "      <td>Biotech &amp; Pharmaceuticals</td>\n",
       "      <td>Biotech &amp; Pharmaceuticals</td>\n",
       "      <td>$2 to $5 billion (USD)</td>\n",
       "      <td>AbbVie</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3909 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Unnamed: 0                                          Job Title  \\\n",
       "index                                                                  \n",
       "0               0                              Senior Data Scientist   \n",
       "1               1                  Data Scientist, Product Analytics   \n",
       "2               2                               Data Science Manager   \n",
       "3               3                                       Data Analyst   \n",
       "4               4                             Director, Data Science   \n",
       "...           ...                                                ...   \n",
       "4375         3904                                  AWS Data Engineer   \n",
       "4376         3905                              Data Analyst â Junior   \n",
       "4377         3906                   Security Analytics Data Engineer   \n",
       "4378         3907                   Security Analytics Data Engineer   \n",
       "4379         3908  Patient Safety Physician or Safety Scientist -...   \n",
       "\n",
       "                    Salary Estimate  \\\n",
       "index                                 \n",
       "0      $111K-$181K (Glassdoor est.)   \n",
       "1      $111K-$181K (Glassdoor est.)   \n",
       "2      $111K-$181K (Glassdoor est.)   \n",
       "3      $111K-$181K (Glassdoor est.)   \n",
       "4      $111K-$181K (Glassdoor est.)   \n",
       "...                             ...   \n",
       "4375    $55K-$112K (Glassdoor est.)   \n",
       "4376    $55K-$112K (Glassdoor est.)   \n",
       "4377    $55K-$112K (Glassdoor est.)   \n",
       "4378    $55K-$112K (Glassdoor est.)   \n",
       "4379    $55K-$112K (Glassdoor est.)   \n",
       "\n",
       "                                         Job Description  Rating  \\\n",
       "index                                                              \n",
       "0      ABOUT HOPPER\\n\\nAt Hopper, we’re on a mission ...     3.5   \n",
       "1      At Noom, we use scientifically proven methods ...     4.5   \n",
       "2      Decode_M\\n\\nhttps://www.decode-m.com/\\n\\nData ...    -1.0   \n",
       "3      Sapphire Digital seeks a dynamic and driven mi...     3.4   \n",
       "4      Director, Data Science - (200537)\\nDescription...     3.4   \n",
       "...                                                  ...     ...   \n",
       "4375   About Us\\n\\nTachyon Technologies is a Digital ...     4.4   \n",
       "4376   Job description\\nInterpret data, analyze resul...     5.0   \n",
       "4377   Job DescriptionThe Security Analytics Data Eng...     3.8   \n",
       "4378   The Security Analytics Data Engineer will inte...     4.0   \n",
       "4379   Help us transform patients' lives.\\nAt UCB, we...     3.7   \n",
       "\n",
       "                                Company Name       Location      Headquarters  \\\n",
       "index                                                                           \n",
       "0                                Hopper\\n3.5   New York, NY  Montreal, Canada   \n",
       "1                               Noom US\\n4.5   New York, NY      New York, NY   \n",
       "2                                   Decode_M   New York, NY      New York, NY   \n",
       "3                      Sapphire Digital\\n3.4  Lyndhurst, NJ     Lyndhurst, NJ   \n",
       "4            United Entertainment Group\\n3.4   New York, NY      New York, NY   \n",
       "...                                      ...            ...               ...   \n",
       "4375               Tachyon Technologies\\n4.4     Dublin, OH        Irving, TX   \n",
       "4376   Staffigo Technical Services, LLC\\n5.0   Columbus, OH     Woodridge, IL   \n",
       "4377                     PDS Tech, Inc.\\n3.8     Dublin, OH        Irving, TX   \n",
       "4378         Data Resource Technologies\\n4.0     Dublin, OH         Omaha, NE   \n",
       "4379                                UCB\\n3.7     Slough, OH  Brussel, Belgium   \n",
       "\n",
       "                          Size  Founded  Type of ownership  \\\n",
       "index                                                        \n",
       "0        501 to 1000 employees     2007  Company - Private   \n",
       "1       1001 to 5000 employees     2008  Company - Private   \n",
       "2            1 to 50 employees       -1            Unknown   \n",
       "3         201 to 500 employees     2019  Company - Private   \n",
       "4          51 to 200 employees     2007  Company - Private   \n",
       "...                        ...      ...                ...   \n",
       "4375      201 to 500 employees     2011  Company - Private   \n",
       "4376       51 to 200 employees     2008  Company - Private   \n",
       "4377   5001 to 10000 employees     1977  Company - Private   \n",
       "4378         1 to 50 employees       -1  Company - Private   \n",
       "4379   5001 to 10000 employees       -1   Company - Public   \n",
       "\n",
       "                        Industry                     Sector  \\\n",
       "index                                                         \n",
       "0                Travel Agencies           Travel & Tourism   \n",
       "1      Health, Beauty, & Fitness          Consumer Services   \n",
       "2                             -1                         -1   \n",
       "3                       Internet     Information Technology   \n",
       "4        Advertising & Marketing          Business Services   \n",
       "...                          ...                        ...   \n",
       "4375                 IT Services     Information Technology   \n",
       "4376                 IT Services     Information Technology   \n",
       "4377      Staffing & Outsourcing          Business Services   \n",
       "4378                  Accounting         Accounting & Legal   \n",
       "4379   Biotech & Pharmaceuticals  Biotech & Pharmaceuticals   \n",
       "\n",
       "                          Revenue               Competitors Easy Apply  \n",
       "index                                                                   \n",
       "0        Unknown / Non-Applicable                        -1         -1  \n",
       "1        Unknown / Non-Applicable                        -1         -1  \n",
       "2        Unknown / Non-Applicable                        -1       True  \n",
       "3        Unknown / Non-Applicable      Zocdoc, Healthgrades         -1  \n",
       "4        Unknown / Non-Applicable  BBDO, Grey Group, Droga5         -1  \n",
       "...                           ...                       ...        ...  \n",
       "4375     $10 to $25 million (USD)                        -1         -1  \n",
       "4376    $50 to $100 million (USD)                        -1         -1  \n",
       "4377   $100 to $500 million (USD)                        -1         -1  \n",
       "4378   Less than $1 million (USD)                        -1         -1  \n",
       "4379       $2 to $5 billion (USD)                    AbbVie         -1  \n",
       "\n",
       "[3909 rows x 16 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4c4a283e-f6dc-4bc3-bff9-7bd5f11f6b71",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', 'Job Title', 'Salary Estimate', 'Job Description',\n",
       "       'Rating', 'Company Name', 'Location', 'Headquarters', 'Size', 'Founded',\n",
       "       'Type of ownership', 'Industry', 'Sector', 'Revenue', 'Competitors',\n",
       "       'Easy Apply'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "380aa2a7-8b87-4a31-a336-5c2c269b6e99",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop(columns='Unnamed: 0', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5ceae4e6-35b0-4536-8d50-26bc2a402b62",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Job Title</th>\n",
       "      <th>Salary Estimate</th>\n",
       "      <th>Job Description</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Company Name</th>\n",
       "      <th>Location</th>\n",
       "      <th>Headquarters</th>\n",
       "      <th>Size</th>\n",
       "      <th>Founded</th>\n",
       "      <th>Type of ownership</th>\n",
       "      <th>Industry</th>\n",
       "      <th>Sector</th>\n",
       "      <th>Revenue</th>\n",
       "      <th>Competitors</th>\n",
       "      <th>Easy Apply</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Senior Data Scientist</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>ABOUT HOPPER\\n\\nAt Hopper, we’re on a mission ...</td>\n",
       "      <td>3.5</td>\n",
       "      <td>Hopper\\n3.5</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>Montreal, Canada</td>\n",
       "      <td>501 to 1000 employees</td>\n",
       "      <td>2007</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Travel Agencies</td>\n",
       "      <td>Travel &amp; Tourism</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Data Scientist, Product Analytics</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>At Noom, we use scientifically proven methods ...</td>\n",
       "      <td>4.5</td>\n",
       "      <td>Noom US\\n4.5</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>1001 to 5000 employees</td>\n",
       "      <td>2008</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Health, Beauty, &amp; Fitness</td>\n",
       "      <td>Consumer Services</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Data Science Manager</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>Decode_M\\n\\nhttps://www.decode-m.com/\\n\\nData ...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>Decode_M</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>1 to 50 employees</td>\n",
       "      <td>-1</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               Job Title               Salary Estimate  \\\n",
       "index                                                                    \n",
       "0                  Senior Data Scientist  $111K-$181K (Glassdoor est.)   \n",
       "1      Data Scientist, Product Analytics  $111K-$181K (Glassdoor est.)   \n",
       "2                   Data Science Manager  $111K-$181K (Glassdoor est.)   \n",
       "\n",
       "                                         Job Description  Rating  \\\n",
       "index                                                              \n",
       "0      ABOUT HOPPER\\n\\nAt Hopper, we’re on a mission ...     3.5   \n",
       "1      At Noom, we use scientifically proven methods ...     4.5   \n",
       "2      Decode_M\\n\\nhttps://www.decode-m.com/\\n\\nData ...    -1.0   \n",
       "\n",
       "       Company Name      Location      Headquarters                    Size  \\\n",
       "index                                                                         \n",
       "0       Hopper\\n3.5  New York, NY  Montreal, Canada   501 to 1000 employees   \n",
       "1      Noom US\\n4.5  New York, NY      New York, NY  1001 to 5000 employees   \n",
       "2          Decode_M  New York, NY      New York, NY       1 to 50 employees   \n",
       "\n",
       "       Founded  Type of ownership                   Industry  \\\n",
       "index                                                          \n",
       "0         2007  Company - Private            Travel Agencies   \n",
       "1         2008  Company - Private  Health, Beauty, & Fitness   \n",
       "2           -1            Unknown                         -1   \n",
       "\n",
       "                  Sector                   Revenue Competitors Easy Apply  \n",
       "index                                                                      \n",
       "0       Travel & Tourism  Unknown / Non-Applicable          -1         -1  \n",
       "1      Consumer Services  Unknown / Non-Applicable          -1         -1  \n",
       "2                     -1  Unknown / Non-Applicable          -1       True  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5a30b670-244d-4fc8-8244-a5074854748d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3909, 15)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "352b42dd-2e32-4dee-849a-4b8d457dd871",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 3909 entries, 0 to 4379\n",
      "Data columns (total 15 columns):\n",
      " #   Column             Non-Null Count  Dtype  \n",
      "---  ------             --------------  -----  \n",
      " 0   Job Title          3909 non-null   object \n",
      " 1   Salary Estimate    3909 non-null   object \n",
      " 2   Job Description    3909 non-null   object \n",
      " 3   Rating             3909 non-null   float64\n",
      " 4   Company Name       3909 non-null   object \n",
      " 5   Location           3909 non-null   object \n",
      " 6   Headquarters       3909 non-null   object \n",
      " 7   Size               3909 non-null   object \n",
      " 8   Founded            3909 non-null   int64  \n",
      " 9   Type of ownership  3909 non-null   object \n",
      " 10  Industry           3909 non-null   object \n",
      " 11  Sector             3909 non-null   object \n",
      " 12  Revenue            3909 non-null   object \n",
      " 13  Competitors        3909 non-null   object \n",
      " 14  Easy Apply         3909 non-null   object \n",
      "dtypes: float64(1), int64(1), object(13)\n",
      "memory usage: 488.6+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3edc992a-0c41-4897-a2dc-b72ed096a944",
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = pd.read_csv('dataset/DataAnalyst.csv', encoding=\"latin\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e12f0989-d8e8-4d23-af71-6b20679eadf5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Job Title</th>\n",
       "      <th>Salary Estimate</th>\n",
       "      <th>Job Description</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Company Name</th>\n",
       "      <th>Location</th>\n",
       "      <th>Headquarters</th>\n",
       "      <th>Size</th>\n",
       "      <th>Founded</th>\n",
       "      <th>Type of ownership</th>\n",
       "      <th>Industry</th>\n",
       "      <th>Sector</th>\n",
       "      <th>Revenue</th>\n",
       "      <th>Competitors</th>\n",
       "      <th>Easy Apply</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Data Analyst, Center on Immigration and Justic...</td>\n",
       "      <td>$37K-$66K (Glassdoor est.)</td>\n",
       "      <td>Are you eager to roll up your sleeves and harn...</td>\n",
       "      <td>3.2</td>\n",
       "      <td>Vera Institute of Justice\\n3.2</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>201 to 500 employees</td>\n",
       "      <td>1961</td>\n",
       "      <td>Nonprofit Organization</td>\n",
       "      <td>Social Assistance</td>\n",
       "      <td>Non-Profit</td>\n",
       "      <td>$100 to $500 million (USD)</td>\n",
       "      <td>-1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Quality Data Analyst</td>\n",
       "      <td>$37K-$66K (Glassdoor est.)</td>\n",
       "      <td>Overview\\n\\nProvides analytical and technical ...</td>\n",
       "      <td>3.8</td>\n",
       "      <td>Visiting Nurse Service of New York\\n3.8</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>1893</td>\n",
       "      <td>Nonprofit Organization</td>\n",
       "      <td>Health Care Services &amp; Hospitals</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>$2 to $5 billion (USD)</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Senior Data Analyst, Insights &amp; Analytics Team...</td>\n",
       "      <td>$37K-$66K (Glassdoor est.)</td>\n",
       "      <td>Weâre looking for a Senior Data Analyst who ...</td>\n",
       "      <td>3.4</td>\n",
       "      <td>Squarespace\\n3.4</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>1001 to 5000 employees</td>\n",
       "      <td>2003</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Internet</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>GoDaddy</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Data Analyst</td>\n",
       "      <td>$37K-$66K (Glassdoor est.)</td>\n",
       "      <td>Requisition NumberRR-0001939\\nRemote:Yes\\nWe c...</td>\n",
       "      <td>4.1</td>\n",
       "      <td>Celerity\\n4.1</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>McLean, VA</td>\n",
       "      <td>201 to 500 employees</td>\n",
       "      <td>2002</td>\n",
       "      <td>Subsidiary or Business Segment</td>\n",
       "      <td>IT Services</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>$50 to $100 million (USD)</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Reporting Data Analyst</td>\n",
       "      <td>$37K-$66K (Glassdoor est.)</td>\n",
       "      <td>ABOUT FANDUEL GROUP\\n\\nFanDuel Group is a worl...</td>\n",
       "      <td>3.9</td>\n",
       "      <td>FanDuel\\n3.9</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>501 to 1000 employees</td>\n",
       "      <td>2009</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Sports &amp; Recreation</td>\n",
       "      <td>Arts, Entertainment &amp; Recreation</td>\n",
       "      <td>$100 to $500 million (USD)</td>\n",
       "      <td>DraftKings</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>Data Analyst</td>\n",
       "      <td>$37K-$66K (Glassdoor est.)</td>\n",
       "      <td>About Cubist\\nCubist Systematic Strategies is ...</td>\n",
       "      <td>3.9</td>\n",
       "      <td>Point72\\n3.9</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>Stamford, CT</td>\n",
       "      <td>1001 to 5000 employees</td>\n",
       "      <td>2014</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Investment Banking &amp; Asset Management</td>\n",
       "      <td>Finance</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>Business/Data Analyst (FP&amp;A)</td>\n",
       "      <td>$37K-$66K (Glassdoor est.)</td>\n",
       "      <td>Two Sigma is a different kind of investment ma...</td>\n",
       "      <td>4.4</td>\n",
       "      <td>Two Sigma\\n4.4</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>1001 to 5000 employees</td>\n",
       "      <td>2001</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Investment Banking &amp; Asset Management</td>\n",
       "      <td>Finance</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>Data Science Analyst</td>\n",
       "      <td>$37K-$66K (Glassdoor est.)</td>\n",
       "      <td>Data Science Analyst\\n\\nJob Details\\nLevel\\nEx...</td>\n",
       "      <td>3.7</td>\n",
       "      <td>GNY Insurance Companies\\n3.7</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>201 to 500 employees</td>\n",
       "      <td>1914</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Insurance Carriers</td>\n",
       "      <td>Insurance</td>\n",
       "      <td>$100 to $500 million (USD)</td>\n",
       "      <td>Travelers, Chubb, Crum &amp; Forster</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>Data Analyst</td>\n",
       "      <td>$37K-$66K (Glassdoor est.)</td>\n",
       "      <td>The Data Analyst is an integral member of the ...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>DMGT\\n4.0</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>London, United Kingdom</td>\n",
       "      <td>5001 to 10000 employees</td>\n",
       "      <td>1896</td>\n",
       "      <td>Company - Public</td>\n",
       "      <td>Venture Capital &amp; Private Equity</td>\n",
       "      <td>Finance</td>\n",
       "      <td>$1 to $2 billion (USD)</td>\n",
       "      <td>Thomson Reuters, Hearst, Pearson</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>Data Analyst, Merchant Health</td>\n",
       "      <td>$37K-$66K (Glassdoor est.)</td>\n",
       "      <td>About Us\\n\\nRiskified is the AI platform power...</td>\n",
       "      <td>4.4</td>\n",
       "      <td>Riskified\\n4.4</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>501 to 1000 employees</td>\n",
       "      <td>2013</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Research &amp; Development</td>\n",
       "      <td>Business Services</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>Signifyd, Forter</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                          Job Title  \\\n",
       "0           0  Data Analyst, Center on Immigration and Justic...   \n",
       "1           1                               Quality Data Analyst   \n",
       "2           2  Senior Data Analyst, Insights & Analytics Team...   \n",
       "3           3                                       Data Analyst   \n",
       "4           4                             Reporting Data Analyst   \n",
       "5           5                                       Data Analyst   \n",
       "6           6                       Business/Data Analyst (FP&A)   \n",
       "7           7                               Data Science Analyst   \n",
       "8           8                                       Data Analyst   \n",
       "9           9                      Data Analyst, Merchant Health   \n",
       "\n",
       "              Salary Estimate  \\\n",
       "0  $37K-$66K (Glassdoor est.)   \n",
       "1  $37K-$66K (Glassdoor est.)   \n",
       "2  $37K-$66K (Glassdoor est.)   \n",
       "3  $37K-$66K (Glassdoor est.)   \n",
       "4  $37K-$66K (Glassdoor est.)   \n",
       "5  $37K-$66K (Glassdoor est.)   \n",
       "6  $37K-$66K (Glassdoor est.)   \n",
       "7  $37K-$66K (Glassdoor est.)   \n",
       "8  $37K-$66K (Glassdoor est.)   \n",
       "9  $37K-$66K (Glassdoor est.)   \n",
       "\n",
       "                                     Job Description  Rating  \\\n",
       "0  Are you eager to roll up your sleeves and harn...     3.2   \n",
       "1  Overview\\n\\nProvides analytical and technical ...     3.8   \n",
       "2  Weâre looking for a Senior Data Analyst who ...     3.4   \n",
       "3  Requisition NumberRR-0001939\\nRemote:Yes\\nWe c...     4.1   \n",
       "4  ABOUT FANDUEL GROUP\\n\\nFanDuel Group is a worl...     3.9   \n",
       "5  About Cubist\\nCubist Systematic Strategies is ...     3.9   \n",
       "6  Two Sigma is a different kind of investment ma...     4.4   \n",
       "7  Data Science Analyst\\n\\nJob Details\\nLevel\\nEx...     3.7   \n",
       "8  The Data Analyst is an integral member of the ...     4.0   \n",
       "9  About Us\\n\\nRiskified is the AI platform power...     4.4   \n",
       "\n",
       "                              Company Name      Location  \\\n",
       "0           Vera Institute of Justice\\n3.2  New York, NY   \n",
       "1  Visiting Nurse Service of New York\\n3.8  New York, NY   \n",
       "2                         Squarespace\\n3.4  New York, NY   \n",
       "3                            Celerity\\n4.1  New York, NY   \n",
       "4                             FanDuel\\n3.9  New York, NY   \n",
       "5                             Point72\\n3.9  New York, NY   \n",
       "6                           Two Sigma\\n4.4  New York, NY   \n",
       "7             GNY Insurance Companies\\n3.7  New York, NY   \n",
       "8                                DMGT\\n4.0  New York, NY   \n",
       "9                           Riskified\\n4.4  New York, NY   \n",
       "\n",
       "             Headquarters                     Size  Founded  \\\n",
       "0            New York, NY     201 to 500 employees     1961   \n",
       "1            New York, NY         10000+ employees     1893   \n",
       "2            New York, NY   1001 to 5000 employees     2003   \n",
       "3              McLean, VA     201 to 500 employees     2002   \n",
       "4            New York, NY    501 to 1000 employees     2009   \n",
       "5            Stamford, CT   1001 to 5000 employees     2014   \n",
       "6            New York, NY   1001 to 5000 employees     2001   \n",
       "7            New York, NY     201 to 500 employees     1914   \n",
       "8  London, United Kingdom  5001 to 10000 employees     1896   \n",
       "9            New York, NY    501 to 1000 employees     2013   \n",
       "\n",
       "                Type of ownership                               Industry  \\\n",
       "0          Nonprofit Organization                      Social Assistance   \n",
       "1          Nonprofit Organization       Health Care Services & Hospitals   \n",
       "2               Company - Private                               Internet   \n",
       "3  Subsidiary or Business Segment                            IT Services   \n",
       "4               Company - Private                    Sports & Recreation   \n",
       "5               Company - Private  Investment Banking & Asset Management   \n",
       "6               Company - Private  Investment Banking & Asset Management   \n",
       "7               Company - Private                     Insurance Carriers   \n",
       "8                Company - Public       Venture Capital & Private Equity   \n",
       "9               Company - Private                 Research & Development   \n",
       "\n",
       "                             Sector                     Revenue  \\\n",
       "0                        Non-Profit  $100 to $500 million (USD)   \n",
       "1                       Health Care      $2 to $5 billion (USD)   \n",
       "2            Information Technology    Unknown / Non-Applicable   \n",
       "3            Information Technology   $50 to $100 million (USD)   \n",
       "4  Arts, Entertainment & Recreation  $100 to $500 million (USD)   \n",
       "5                           Finance    Unknown / Non-Applicable   \n",
       "6                           Finance    Unknown / Non-Applicable   \n",
       "7                         Insurance  $100 to $500 million (USD)   \n",
       "8                           Finance      $1 to $2 billion (USD)   \n",
       "9                 Business Services    Unknown / Non-Applicable   \n",
       "\n",
       "                        Competitors Easy Apply  \n",
       "0                                -1       True  \n",
       "1                                -1         -1  \n",
       "2                           GoDaddy         -1  \n",
       "3                                -1         -1  \n",
       "4                        DraftKings       True  \n",
       "5                                -1         -1  \n",
       "6                                -1         -1  \n",
       "7  Travelers, Chubb, Crum & Forster       True  \n",
       "8  Thomson Reuters, Hearst, Pearson         -1  \n",
       "9                  Signifyd, Forter         -1  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2489cffe-23ea-4982-9040-78eddd12b840",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2253, 16)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "aafa9432-b2d7-484f-820e-eff5fcd728a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', 'Job Title', 'Salary Estimate', 'Job Description',\n",
       "       'Rating', 'Company Name', 'Location', 'Headquarters', 'Size', 'Founded',\n",
       "       'Type of ownership', 'Industry', 'Sector', 'Revenue', 'Competitors',\n",
       "       'Easy Apply'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b3e780d3-b789-4833-bc23-112f014f577b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>Job Title</th>\n",
       "      <th>Salary Estimate</th>\n",
       "      <th>Job Description</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Company Name</th>\n",
       "      <th>Location</th>\n",
       "      <th>Headquarters</th>\n",
       "      <th>Size</th>\n",
       "      <th>Founded</th>\n",
       "      <th>Type of ownership</th>\n",
       "      <th>Industry</th>\n",
       "      <th>Sector</th>\n",
       "      <th>Revenue</th>\n",
       "      <th>Competitors</th>\n",
       "      <th>Easy Apply</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Data Analyst, Center on Immigration and Justic...</td>\n",
       "      <td>$37K-$66K (Glassdoor est.)</td>\n",
       "      <td>Are you eager to roll up your sleeves and harn...</td>\n",
       "      <td>3.2</td>\n",
       "      <td>Vera Institute of Justice\\n3.2</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>201 to 500 employees</td>\n",
       "      <td>1961</td>\n",
       "      <td>Nonprofit Organization</td>\n",
       "      <td>Social Assistance</td>\n",
       "      <td>Non-Profit</td>\n",
       "      <td>$100 to $500 million (USD)</td>\n",
       "      <td>-1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Quality Data Analyst</td>\n",
       "      <td>$37K-$66K (Glassdoor est.)</td>\n",
       "      <td>Overview\\n\\nProvides analytical and technical ...</td>\n",
       "      <td>3.8</td>\n",
       "      <td>Visiting Nurse Service of New York\\n3.8</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>1893</td>\n",
       "      <td>Nonprofit Organization</td>\n",
       "      <td>Health Care Services &amp; Hospitals</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>$2 to $5 billion (USD)</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Senior Data Analyst, Insights &amp; Analytics Team...</td>\n",
       "      <td>$37K-$66K (Glassdoor est.)</td>\n",
       "      <td>Weâre looking for a Senior Data Analyst who ...</td>\n",
       "      <td>3.4</td>\n",
       "      <td>Squarespace\\n3.4</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>1001 to 5000 employees</td>\n",
       "      <td>2003</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Internet</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>GoDaddy</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Data Analyst</td>\n",
       "      <td>$37K-$66K (Glassdoor est.)</td>\n",
       "      <td>Requisition NumberRR-0001939\\nRemote:Yes\\nWe c...</td>\n",
       "      <td>4.1</td>\n",
       "      <td>Celerity\\n4.1</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>McLean, VA</td>\n",
       "      <td>201 to 500 employees</td>\n",
       "      <td>2002</td>\n",
       "      <td>Subsidiary or Business Segment</td>\n",
       "      <td>IT Services</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>$50 to $100 million (USD)</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Reporting Data Analyst</td>\n",
       "      <td>$37K-$66K (Glassdoor est.)</td>\n",
       "      <td>ABOUT FANDUEL GROUP\\n\\nFanDuel Group is a worl...</td>\n",
       "      <td>3.9</td>\n",
       "      <td>FanDuel\\n3.9</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>501 to 1000 employees</td>\n",
       "      <td>2009</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Sports &amp; Recreation</td>\n",
       "      <td>Arts, Entertainment &amp; Recreation</td>\n",
       "      <td>$100 to $500 million (USD)</td>\n",
       "      <td>DraftKings</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2248</th>\n",
       "      <td>2248</td>\n",
       "      <td>RQS - IHHA - 201900004460 -1q Data Security An...</td>\n",
       "      <td>$78K-$104K (Glassdoor est.)</td>\n",
       "      <td>Maintains systems to protect data from unautho...</td>\n",
       "      <td>2.5</td>\n",
       "      <td>Avacend, Inc.\\n2.5</td>\n",
       "      <td>Denver, CO</td>\n",
       "      <td>Alpharetta, GA</td>\n",
       "      <td>51 to 200 employees</td>\n",
       "      <td>-1</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Staffing &amp; Outsourcing</td>\n",
       "      <td>Business Services</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2249</th>\n",
       "      <td>2249</td>\n",
       "      <td>Senior Data Analyst (Corporate Audit)</td>\n",
       "      <td>$78K-$104K (Glassdoor est.)</td>\n",
       "      <td>Position:\\nSenior Data Analyst (Corporate Audi...</td>\n",
       "      <td>2.9</td>\n",
       "      <td>Arrow Electronics\\n2.9</td>\n",
       "      <td>Centennial, CO</td>\n",
       "      <td>Centennial, CO</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>1935</td>\n",
       "      <td>Company - Public</td>\n",
       "      <td>Wholesale</td>\n",
       "      <td>Business Services</td>\n",
       "      <td>$10+ billion (USD)</td>\n",
       "      <td>Avnet, Ingram Micro, Tech Data</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2250</th>\n",
       "      <td>2250</td>\n",
       "      <td>Technical Business Analyst (SQL, Data analytic...</td>\n",
       "      <td>$78K-$104K (Glassdoor est.)</td>\n",
       "      <td>Title: Technical Business Analyst (SQL, Data a...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>Spiceorb</td>\n",
       "      <td>Denver, CO</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2251</th>\n",
       "      <td>2251</td>\n",
       "      <td>Data Analyst 3, Customer Experience</td>\n",
       "      <td>$78K-$104K (Glassdoor est.)</td>\n",
       "      <td>Summary\\n\\nResponsible for working cross-funct...</td>\n",
       "      <td>3.1</td>\n",
       "      <td>Contingent Network Services\\n3.1</td>\n",
       "      <td>Centennial, CO</td>\n",
       "      <td>West Chester, OH</td>\n",
       "      <td>201 to 500 employees</td>\n",
       "      <td>1984</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Enterprise Software &amp; Network Solutions</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>$25 to $50 million (USD)</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2252</th>\n",
       "      <td>2252</td>\n",
       "      <td>Senior Quality Data Analyst</td>\n",
       "      <td>$78K-$104K (Glassdoor est.)</td>\n",
       "      <td>You.\\n\\nYou bring your body, mind, heart and s...</td>\n",
       "      <td>3.4</td>\n",
       "      <td>SCL Health\\n3.4</td>\n",
       "      <td>Broomfield, CO</td>\n",
       "      <td>Broomfield, CO</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>1864</td>\n",
       "      <td>Nonprofit Organization</td>\n",
       "      <td>Health Care Services &amp; Hospitals</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>$2 to $5 billion (USD)</td>\n",
       "      <td>Centura Health, HealthONE, Denver Health and H...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2253 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      index                                          Job Title  \\\n",
       "0         0  Data Analyst, Center on Immigration and Justic...   \n",
       "1         1                               Quality Data Analyst   \n",
       "2         2  Senior Data Analyst, Insights & Analytics Team...   \n",
       "3         3                                       Data Analyst   \n",
       "4         4                             Reporting Data Analyst   \n",
       "...     ...                                                ...   \n",
       "2248   2248  RQS - IHHA - 201900004460 -1q Data Security An...   \n",
       "2249   2249              Senior Data Analyst (Corporate Audit)   \n",
       "2250   2250  Technical Business Analyst (SQL, Data analytic...   \n",
       "2251   2251                Data Analyst 3, Customer Experience   \n",
       "2252   2252                        Senior Quality Data Analyst   \n",
       "\n",
       "                  Salary Estimate  \\\n",
       "0      $37K-$66K (Glassdoor est.)   \n",
       "1      $37K-$66K (Glassdoor est.)   \n",
       "2      $37K-$66K (Glassdoor est.)   \n",
       "3      $37K-$66K (Glassdoor est.)   \n",
       "4      $37K-$66K (Glassdoor est.)   \n",
       "...                           ...   \n",
       "2248  $78K-$104K (Glassdoor est.)   \n",
       "2249  $78K-$104K (Glassdoor est.)   \n",
       "2250  $78K-$104K (Glassdoor est.)   \n",
       "2251  $78K-$104K (Glassdoor est.)   \n",
       "2252  $78K-$104K (Glassdoor est.)   \n",
       "\n",
       "                                        Job Description  Rating  \\\n",
       "0     Are you eager to roll up your sleeves and harn...     3.2   \n",
       "1     Overview\\n\\nProvides analytical and technical ...     3.8   \n",
       "2     Weâre looking for a Senior Data Analyst who ...     3.4   \n",
       "3     Requisition NumberRR-0001939\\nRemote:Yes\\nWe c...     4.1   \n",
       "4     ABOUT FANDUEL GROUP\\n\\nFanDuel Group is a worl...     3.9   \n",
       "...                                                 ...     ...   \n",
       "2248  Maintains systems to protect data from unautho...     2.5   \n",
       "2249  Position:\\nSenior Data Analyst (Corporate Audi...     2.9   \n",
       "2250  Title: Technical Business Analyst (SQL, Data a...    -1.0   \n",
       "2251  Summary\\n\\nResponsible for working cross-funct...     3.1   \n",
       "2252  You.\\n\\nYou bring your body, mind, heart and s...     3.4   \n",
       "\n",
       "                                 Company Name        Location  \\\n",
       "0              Vera Institute of Justice\\n3.2    New York, NY   \n",
       "1     Visiting Nurse Service of New York\\n3.8    New York, NY   \n",
       "2                            Squarespace\\n3.4    New York, NY   \n",
       "3                               Celerity\\n4.1    New York, NY   \n",
       "4                                FanDuel\\n3.9    New York, NY   \n",
       "...                                       ...             ...   \n",
       "2248                       Avacend, Inc.\\n2.5      Denver, CO   \n",
       "2249                   Arrow Electronics\\n2.9  Centennial, CO   \n",
       "2250                                 Spiceorb      Denver, CO   \n",
       "2251         Contingent Network Services\\n3.1  Centennial, CO   \n",
       "2252                          SCL Health\\n3.4  Broomfield, CO   \n",
       "\n",
       "          Headquarters                    Size  Founded  \\\n",
       "0         New York, NY    201 to 500 employees     1961   \n",
       "1         New York, NY        10000+ employees     1893   \n",
       "2         New York, NY  1001 to 5000 employees     2003   \n",
       "3           McLean, VA    201 to 500 employees     2002   \n",
       "4         New York, NY   501 to 1000 employees     2009   \n",
       "...                ...                     ...      ...   \n",
       "2248    Alpharetta, GA     51 to 200 employees       -1   \n",
       "2249    Centennial, CO        10000+ employees     1935   \n",
       "2250                -1                      -1       -1   \n",
       "2251  West Chester, OH    201 to 500 employees     1984   \n",
       "2252    Broomfield, CO        10000+ employees     1864   \n",
       "\n",
       "                   Type of ownership                                 Industry  \\\n",
       "0             Nonprofit Organization                        Social Assistance   \n",
       "1             Nonprofit Organization         Health Care Services & Hospitals   \n",
       "2                  Company - Private                                 Internet   \n",
       "3     Subsidiary or Business Segment                              IT Services   \n",
       "4                  Company - Private                      Sports & Recreation   \n",
       "...                              ...                                      ...   \n",
       "2248               Company - Private                   Staffing & Outsourcing   \n",
       "2249                Company - Public                                Wholesale   \n",
       "2250                              -1                                       -1   \n",
       "2251               Company - Private  Enterprise Software & Network Solutions   \n",
       "2252          Nonprofit Organization         Health Care Services & Hospitals   \n",
       "\n",
       "                                Sector                     Revenue  \\\n",
       "0                           Non-Profit  $100 to $500 million (USD)   \n",
       "1                          Health Care      $2 to $5 billion (USD)   \n",
       "2               Information Technology    Unknown / Non-Applicable   \n",
       "3               Information Technology   $50 to $100 million (USD)   \n",
       "4     Arts, Entertainment & Recreation  $100 to $500 million (USD)   \n",
       "...                                ...                         ...   \n",
       "2248                 Business Services    Unknown / Non-Applicable   \n",
       "2249                 Business Services          $10+ billion (USD)   \n",
       "2250                                -1                          -1   \n",
       "2251            Information Technology    $25 to $50 million (USD)   \n",
       "2252                       Health Care      $2 to $5 billion (USD)   \n",
       "\n",
       "                                            Competitors Easy Apply  \n",
       "0                                                    -1       True  \n",
       "1                                                    -1         -1  \n",
       "2                                               GoDaddy         -1  \n",
       "3                                                    -1         -1  \n",
       "4                                            DraftKings       True  \n",
       "...                                                 ...        ...  \n",
       "2248                                                 -1         -1  \n",
       "2249                     Avnet, Ingram Micro, Tech Data         -1  \n",
       "2250                                                 -1         -1  \n",
       "2251                                                 -1         -1  \n",
       "2252  Centura Health, HealthONE, Denver Health and H...         -1  \n",
       "\n",
       "[2253 rows x 16 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.rename(columns={'Unnamed: 0': 'index'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6b558abb-e1ef-4c5a-826e-7721b3d9b4f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Job Title</th>\n",
       "      <th>Salary Estimate</th>\n",
       "      <th>Job Description</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Company Name</th>\n",
       "      <th>Location</th>\n",
       "      <th>Headquarters</th>\n",
       "      <th>Size</th>\n",
       "      <th>Founded</th>\n",
       "      <th>Type of ownership</th>\n",
       "      <th>Industry</th>\n",
       "      <th>Sector</th>\n",
       "      <th>Revenue</th>\n",
       "      <th>Competitors</th>\n",
       "      <th>Easy Apply</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Data Analyst, Center on Immigration and Justic...</td>\n",
       "      <td>$37K-$66K (Glassdoor est.)</td>\n",
       "      <td>Are you eager to roll up your sleeves and harn...</td>\n",
       "      <td>3.2</td>\n",
       "      <td>Vera Institute of Justice\\n3.2</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>201 to 500 employees</td>\n",
       "      <td>1961</td>\n",
       "      <td>Nonprofit Organization</td>\n",
       "      <td>Social Assistance</td>\n",
       "      <td>Non-Profit</td>\n",
       "      <td>$100 to $500 million (USD)</td>\n",
       "      <td>-1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Quality Data Analyst</td>\n",
       "      <td>$37K-$66K (Glassdoor est.)</td>\n",
       "      <td>Overview\\n\\nProvides analytical and technical ...</td>\n",
       "      <td>3.8</td>\n",
       "      <td>Visiting Nurse Service of New York\\n3.8</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>1893</td>\n",
       "      <td>Nonprofit Organization</td>\n",
       "      <td>Health Care Services &amp; Hospitals</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>$2 to $5 billion (USD)</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Senior Data Analyst, Insights &amp; Analytics Team...</td>\n",
       "      <td>$37K-$66K (Glassdoor est.)</td>\n",
       "      <td>Weâre looking for a Senior Data Analyst who ...</td>\n",
       "      <td>3.4</td>\n",
       "      <td>Squarespace\\n3.4</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>1001 to 5000 employees</td>\n",
       "      <td>2003</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Internet</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>GoDaddy</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Data Analyst</td>\n",
       "      <td>$37K-$66K (Glassdoor est.)</td>\n",
       "      <td>Requisition NumberRR-0001939\\nRemote:Yes\\nWe c...</td>\n",
       "      <td>4.1</td>\n",
       "      <td>Celerity\\n4.1</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>McLean, VA</td>\n",
       "      <td>201 to 500 employees</td>\n",
       "      <td>2002</td>\n",
       "      <td>Subsidiary or Business Segment</td>\n",
       "      <td>IT Services</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>$50 to $100 million (USD)</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Reporting Data Analyst</td>\n",
       "      <td>$37K-$66K (Glassdoor est.)</td>\n",
       "      <td>ABOUT FANDUEL GROUP\\n\\nFanDuel Group is a worl...</td>\n",
       "      <td>3.9</td>\n",
       "      <td>FanDuel\\n3.9</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>501 to 1000 employees</td>\n",
       "      <td>2009</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Sports &amp; Recreation</td>\n",
       "      <td>Arts, Entertainment &amp; Recreation</td>\n",
       "      <td>$100 to $500 million (USD)</td>\n",
       "      <td>DraftKings</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2248</th>\n",
       "      <td>2248</td>\n",
       "      <td>RQS - IHHA - 201900004460 -1q Data Security An...</td>\n",
       "      <td>$78K-$104K (Glassdoor est.)</td>\n",
       "      <td>Maintains systems to protect data from unautho...</td>\n",
       "      <td>2.5</td>\n",
       "      <td>Avacend, Inc.\\n2.5</td>\n",
       "      <td>Denver, CO</td>\n",
       "      <td>Alpharetta, GA</td>\n",
       "      <td>51 to 200 employees</td>\n",
       "      <td>-1</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Staffing &amp; Outsourcing</td>\n",
       "      <td>Business Services</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2249</th>\n",
       "      <td>2249</td>\n",
       "      <td>Senior Data Analyst (Corporate Audit)</td>\n",
       "      <td>$78K-$104K (Glassdoor est.)</td>\n",
       "      <td>Position:\\nSenior Data Analyst (Corporate Audi...</td>\n",
       "      <td>2.9</td>\n",
       "      <td>Arrow Electronics\\n2.9</td>\n",
       "      <td>Centennial, CO</td>\n",
       "      <td>Centennial, CO</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>1935</td>\n",
       "      <td>Company - Public</td>\n",
       "      <td>Wholesale</td>\n",
       "      <td>Business Services</td>\n",
       "      <td>$10+ billion (USD)</td>\n",
       "      <td>Avnet, Ingram Micro, Tech Data</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2250</th>\n",
       "      <td>2250</td>\n",
       "      <td>Technical Business Analyst (SQL, Data analytic...</td>\n",
       "      <td>$78K-$104K (Glassdoor est.)</td>\n",
       "      <td>Title: Technical Business Analyst (SQL, Data a...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>Spiceorb</td>\n",
       "      <td>Denver, CO</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2251</th>\n",
       "      <td>2251</td>\n",
       "      <td>Data Analyst 3, Customer Experience</td>\n",
       "      <td>$78K-$104K (Glassdoor est.)</td>\n",
       "      <td>Summary\\n\\nResponsible for working cross-funct...</td>\n",
       "      <td>3.1</td>\n",
       "      <td>Contingent Network Services\\n3.1</td>\n",
       "      <td>Centennial, CO</td>\n",
       "      <td>West Chester, OH</td>\n",
       "      <td>201 to 500 employees</td>\n",
       "      <td>1984</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Enterprise Software &amp; Network Solutions</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>$25 to $50 million (USD)</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2252</th>\n",
       "      <td>2252</td>\n",
       "      <td>Senior Quality Data Analyst</td>\n",
       "      <td>$78K-$104K (Glassdoor est.)</td>\n",
       "      <td>You.\\n\\nYou bring your body, mind, heart and s...</td>\n",
       "      <td>3.4</td>\n",
       "      <td>SCL Health\\n3.4</td>\n",
       "      <td>Broomfield, CO</td>\n",
       "      <td>Broomfield, CO</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>1864</td>\n",
       "      <td>Nonprofit Organization</td>\n",
       "      <td>Health Care Services &amp; Hospitals</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>$2 to $5 billion (USD)</td>\n",
       "      <td>Centura Health, HealthONE, Denver Health and H...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2253 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0                                          Job Title  \\\n",
       "0              0  Data Analyst, Center on Immigration and Justic...   \n",
       "1              1                               Quality Data Analyst   \n",
       "2              2  Senior Data Analyst, Insights & Analytics Team...   \n",
       "3              3                                       Data Analyst   \n",
       "4              4                             Reporting Data Analyst   \n",
       "...          ...                                                ...   \n",
       "2248        2248  RQS - IHHA - 201900004460 -1q Data Security An...   \n",
       "2249        2249              Senior Data Analyst (Corporate Audit)   \n",
       "2250        2250  Technical Business Analyst (SQL, Data analytic...   \n",
       "2251        2251                Data Analyst 3, Customer Experience   \n",
       "2252        2252                        Senior Quality Data Analyst   \n",
       "\n",
       "                  Salary Estimate  \\\n",
       "0      $37K-$66K (Glassdoor est.)   \n",
       "1      $37K-$66K (Glassdoor est.)   \n",
       "2      $37K-$66K (Glassdoor est.)   \n",
       "3      $37K-$66K (Glassdoor est.)   \n",
       "4      $37K-$66K (Glassdoor est.)   \n",
       "...                           ...   \n",
       "2248  $78K-$104K (Glassdoor est.)   \n",
       "2249  $78K-$104K (Glassdoor est.)   \n",
       "2250  $78K-$104K (Glassdoor est.)   \n",
       "2251  $78K-$104K (Glassdoor est.)   \n",
       "2252  $78K-$104K (Glassdoor est.)   \n",
       "\n",
       "                                        Job Description  Rating  \\\n",
       "0     Are you eager to roll up your sleeves and harn...     3.2   \n",
       "1     Overview\\n\\nProvides analytical and technical ...     3.8   \n",
       "2     Weâre looking for a Senior Data Analyst who ...     3.4   \n",
       "3     Requisition NumberRR-0001939\\nRemote:Yes\\nWe c...     4.1   \n",
       "4     ABOUT FANDUEL GROUP\\n\\nFanDuel Group is a worl...     3.9   \n",
       "...                                                 ...     ...   \n",
       "2248  Maintains systems to protect data from unautho...     2.5   \n",
       "2249  Position:\\nSenior Data Analyst (Corporate Audi...     2.9   \n",
       "2250  Title: Technical Business Analyst (SQL, Data a...    -1.0   \n",
       "2251  Summary\\n\\nResponsible for working cross-funct...     3.1   \n",
       "2252  You.\\n\\nYou bring your body, mind, heart and s...     3.4   \n",
       "\n",
       "                                 Company Name        Location  \\\n",
       "0              Vera Institute of Justice\\n3.2    New York, NY   \n",
       "1     Visiting Nurse Service of New York\\n3.8    New York, NY   \n",
       "2                            Squarespace\\n3.4    New York, NY   \n",
       "3                               Celerity\\n4.1    New York, NY   \n",
       "4                                FanDuel\\n3.9    New York, NY   \n",
       "...                                       ...             ...   \n",
       "2248                       Avacend, Inc.\\n2.5      Denver, CO   \n",
       "2249                   Arrow Electronics\\n2.9  Centennial, CO   \n",
       "2250                                 Spiceorb      Denver, CO   \n",
       "2251         Contingent Network Services\\n3.1  Centennial, CO   \n",
       "2252                          SCL Health\\n3.4  Broomfield, CO   \n",
       "\n",
       "          Headquarters                    Size  Founded  \\\n",
       "0         New York, NY    201 to 500 employees     1961   \n",
       "1         New York, NY        10000+ employees     1893   \n",
       "2         New York, NY  1001 to 5000 employees     2003   \n",
       "3           McLean, VA    201 to 500 employees     2002   \n",
       "4         New York, NY   501 to 1000 employees     2009   \n",
       "...                ...                     ...      ...   \n",
       "2248    Alpharetta, GA     51 to 200 employees       -1   \n",
       "2249    Centennial, CO        10000+ employees     1935   \n",
       "2250                -1                      -1       -1   \n",
       "2251  West Chester, OH    201 to 500 employees     1984   \n",
       "2252    Broomfield, CO        10000+ employees     1864   \n",
       "\n",
       "                   Type of ownership                                 Industry  \\\n",
       "0             Nonprofit Organization                        Social Assistance   \n",
       "1             Nonprofit Organization         Health Care Services & Hospitals   \n",
       "2                  Company - Private                                 Internet   \n",
       "3     Subsidiary or Business Segment                              IT Services   \n",
       "4                  Company - Private                      Sports & Recreation   \n",
       "...                              ...                                      ...   \n",
       "2248               Company - Private                   Staffing & Outsourcing   \n",
       "2249                Company - Public                                Wholesale   \n",
       "2250                              -1                                       -1   \n",
       "2251               Company - Private  Enterprise Software & Network Solutions   \n",
       "2252          Nonprofit Organization         Health Care Services & Hospitals   \n",
       "\n",
       "                                Sector                     Revenue  \\\n",
       "0                           Non-Profit  $100 to $500 million (USD)   \n",
       "1                          Health Care      $2 to $5 billion (USD)   \n",
       "2               Information Technology    Unknown / Non-Applicable   \n",
       "3               Information Technology   $50 to $100 million (USD)   \n",
       "4     Arts, Entertainment & Recreation  $100 to $500 million (USD)   \n",
       "...                                ...                         ...   \n",
       "2248                 Business Services    Unknown / Non-Applicable   \n",
       "2249                 Business Services          $10+ billion (USD)   \n",
       "2250                                -1                          -1   \n",
       "2251            Information Technology    $25 to $50 million (USD)   \n",
       "2252                       Health Care      $2 to $5 billion (USD)   \n",
       "\n",
       "                                            Competitors Easy Apply  \n",
       "0                                                    -1       True  \n",
       "1                                                    -1         -1  \n",
       "2                                               GoDaddy         -1  \n",
       "3                                                    -1         -1  \n",
       "4                                            DraftKings       True  \n",
       "...                                                 ...        ...  \n",
       "2248                                                 -1         -1  \n",
       "2249                     Avnet, Ingram Micro, Tech Data         -1  \n",
       "2250                                                 -1         -1  \n",
       "2251                                                 -1         -1  \n",
       "2252  Centura Health, HealthONE, Denver Health and H...         -1  \n",
       "\n",
       "[2253 rows x 16 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "ad195988-36e5-4594-9e12-5f200dc5d29a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df2.rename(columns={'Unnamed: 0': 'index'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ec624a74-54ed-4b26-8d67-3f1a20f16de7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>Job Title</th>\n",
       "      <th>Salary Estimate</th>\n",
       "      <th>Job Description</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Company Name</th>\n",
       "      <th>Location</th>\n",
       "      <th>Headquarters</th>\n",
       "      <th>Size</th>\n",
       "      <th>Founded</th>\n",
       "      <th>Type of ownership</th>\n",
       "      <th>Industry</th>\n",
       "      <th>Sector</th>\n",
       "      <th>Revenue</th>\n",
       "      <th>Competitors</th>\n",
       "      <th>Easy Apply</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Data Analyst, Center on Immigration and Justic...</td>\n",
       "      <td>$37K-$66K (Glassdoor est.)</td>\n",
       "      <td>Are you eager to roll up your sleeves and harn...</td>\n",
       "      <td>3.2</td>\n",
       "      <td>Vera Institute of Justice\\n3.2</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>201 to 500 employees</td>\n",
       "      <td>1961</td>\n",
       "      <td>Nonprofit Organization</td>\n",
       "      <td>Social Assistance</td>\n",
       "      <td>Non-Profit</td>\n",
       "      <td>$100 to $500 million (USD)</td>\n",
       "      <td>-1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Quality Data Analyst</td>\n",
       "      <td>$37K-$66K (Glassdoor est.)</td>\n",
       "      <td>Overview\\n\\nProvides analytical and technical ...</td>\n",
       "      <td>3.8</td>\n",
       "      <td>Visiting Nurse Service of New York\\n3.8</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>1893</td>\n",
       "      <td>Nonprofit Organization</td>\n",
       "      <td>Health Care Services &amp; Hospitals</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>$2 to $5 billion (USD)</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Senior Data Analyst, Insights &amp; Analytics Team...</td>\n",
       "      <td>$37K-$66K (Glassdoor est.)</td>\n",
       "      <td>Weâre looking for a Senior Data Analyst who ...</td>\n",
       "      <td>3.4</td>\n",
       "      <td>Squarespace\\n3.4</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>1001 to 5000 employees</td>\n",
       "      <td>2003</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Internet</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>GoDaddy</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index                                          Job Title  \\\n",
       "0      0  Data Analyst, Center on Immigration and Justic...   \n",
       "1      1                               Quality Data Analyst   \n",
       "2      2  Senior Data Analyst, Insights & Analytics Team...   \n",
       "\n",
       "              Salary Estimate  \\\n",
       "0  $37K-$66K (Glassdoor est.)   \n",
       "1  $37K-$66K (Glassdoor est.)   \n",
       "2  $37K-$66K (Glassdoor est.)   \n",
       "\n",
       "                                     Job Description  Rating  \\\n",
       "0  Are you eager to roll up your sleeves and harn...     3.2   \n",
       "1  Overview\\n\\nProvides analytical and technical ...     3.8   \n",
       "2  Weâre looking for a Senior Data Analyst who ...     3.4   \n",
       "\n",
       "                              Company Name      Location  Headquarters  \\\n",
       "0           Vera Institute of Justice\\n3.2  New York, NY  New York, NY   \n",
       "1  Visiting Nurse Service of New York\\n3.8  New York, NY  New York, NY   \n",
       "2                         Squarespace\\n3.4  New York, NY  New York, NY   \n",
       "\n",
       "                     Size  Founded       Type of ownership  \\\n",
       "0    201 to 500 employees     1961  Nonprofit Organization   \n",
       "1        10000+ employees     1893  Nonprofit Organization   \n",
       "2  1001 to 5000 employees     2003       Company - Private   \n",
       "\n",
       "                           Industry                  Sector  \\\n",
       "0                 Social Assistance              Non-Profit   \n",
       "1  Health Care Services & Hospitals             Health Care   \n",
       "2                          Internet  Information Technology   \n",
       "\n",
       "                      Revenue Competitors Easy Apply  \n",
       "0  $100 to $500 million (USD)          -1       True  \n",
       "1      $2 to $5 billion (USD)          -1         -1  \n",
       "2    Unknown / Non-Applicable     GoDaddy         -1  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "d54bb8b0-bbb5-480c-b942-fd5f11cb4c1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df2.set_index('index', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "654b7e7e-25be-4897-8f20-0c5e65270674",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2253, 15)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "198d8bc7-7f15-46d3-af53-714f1f8c8b33",
   "metadata": {},
   "source": [
    "## Dataset \n",
    "Having loaded both dataset (DataAnalyst and DataScientist) into DataFrame, only DataScientist dataset will be use for this project as it has more data."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd1d52c2-a3b6-4288-89a5-1c94d8c462e1",
   "metadata": {},
   "source": [
    "![purple-divider](https://user-images.githubusercontent.com/7065401/52071927-c1cd7100-2562-11e9-908a-dde91ba14e59.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a4f80091-d471-4ea4-8303-86c92558fdb5",
   "metadata": {},
   "source": [
    "__The columns will be prepared in the foolowing order__\n",
    "* Company Name: stripping the rating values\n",
    "* Location: parsing state code\n",
    "* HQ: check if job is in same location or in HQ\n",
    "* Size: Parsing (Min, Max, Na) strip -1 with 0\n",
    "* Founded: age of company strip -1 with 0\n",
    "* Type of ownership: categorize \n",
    "* Industry\n",
    "* Sector\n",
    "* Revenue\n",
    "* Competitors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "2dddd063-1d65-4f0f-bd7a-163b628af6db",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Configure the display settings\n",
    "pd.set_option('display.max_rows', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "a5168f41-5c10-4842-bb3d-2b4015aac307",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Job Title</th>\n",
       "      <th>Salary Estimate</th>\n",
       "      <th>Job Description</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Company Name</th>\n",
       "      <th>Location</th>\n",
       "      <th>Headquarters</th>\n",
       "      <th>Size</th>\n",
       "      <th>Founded</th>\n",
       "      <th>Type of ownership</th>\n",
       "      <th>Industry</th>\n",
       "      <th>Sector</th>\n",
       "      <th>Revenue</th>\n",
       "      <th>Competitors</th>\n",
       "      <th>Easy Apply</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Senior Data Scientist</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>ABOUT HOPPER\\n\\nAt Hopper, we’re on a mission ...</td>\n",
       "      <td>3.5</td>\n",
       "      <td>Hopper\\n3.5</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>Montreal, Canada</td>\n",
       "      <td>501 to 1000 employees</td>\n",
       "      <td>2007</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Travel Agencies</td>\n",
       "      <td>Travel &amp; Tourism</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Data Scientist, Product Analytics</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>At Noom, we use scientifically proven methods ...</td>\n",
       "      <td>4.5</td>\n",
       "      <td>Noom US\\n4.5</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>1001 to 5000 employees</td>\n",
       "      <td>2008</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Health, Beauty, &amp; Fitness</td>\n",
       "      <td>Consumer Services</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Data Science Manager</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>Decode_M\\n\\nhttps://www.decode-m.com/\\n\\nData ...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>Decode_M</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>1 to 50 employees</td>\n",
       "      <td>-1</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Data Analyst</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>Sapphire Digital seeks a dynamic and driven mi...</td>\n",
       "      <td>3.4</td>\n",
       "      <td>Sapphire Digital\\n3.4</td>\n",
       "      <td>Lyndhurst, NJ</td>\n",
       "      <td>Lyndhurst, NJ</td>\n",
       "      <td>201 to 500 employees</td>\n",
       "      <td>2019</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Internet</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>Zocdoc, Healthgrades</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Director, Data Science</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>Director, Data Science - (200537)\\nDescription...</td>\n",
       "      <td>3.4</td>\n",
       "      <td>United Entertainment Group\\n3.4</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>51 to 200 employees</td>\n",
       "      <td>2007</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Advertising &amp; Marketing</td>\n",
       "      <td>Business Services</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>BBDO, Grey Group, Droga5</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>Job Brief\\n\\nThe ideal candidate will have pre...</td>\n",
       "      <td>2.9</td>\n",
       "      <td>IFG Companies\\n2.9</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>Hartford, CT</td>\n",
       "      <td>201 to 500 employees</td>\n",
       "      <td>1985</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Insurance Carriers</td>\n",
       "      <td>Insurance</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>Colony Specialty, Markel, RLI</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Quantitative Researcher</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>Experience: Entry-level (PhD Program) or Exper...</td>\n",
       "      <td>4.4</td>\n",
       "      <td>PDT Partners\\n4.4</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>51 to 200 employees</td>\n",
       "      <td>1993</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Investment Banking &amp; Asset Management</td>\n",
       "      <td>Finance</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Quantitative Research Associate</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>Seeking a quant to work with senior researcher...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>Enlightenment Research</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>1 to 50 employees</td>\n",
       "      <td>-1</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>AI Scientist</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>Paige is a software company helping pathologis...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Paige\\n5.0</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>1 to 50 employees</td>\n",
       "      <td>2018</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Enterprise Software &amp; Network Solutions</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Quantitative Researcher</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>About the Position\\n\\n\\nAt Jane Street, we con...</td>\n",
       "      <td>4.8</td>\n",
       "      <td>Jane Street\\n4.8</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>501 to 1000 employees</td>\n",
       "      <td>2000</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Investment Banking &amp; Asset Management</td>\n",
       "      <td>Finance</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>Company Description:\\n\\nQuartet is a pioneerin...</td>\n",
       "      <td>3.9</td>\n",
       "      <td>Quartet Health\\n3.9</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>201 to 500 employees</td>\n",
       "      <td>2014</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Enterprise Software &amp; Network Solutions</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Data Scientist/Machine Learning</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>PulsePoint™, a global programmatic advertising...</td>\n",
       "      <td>4.4</td>\n",
       "      <td>PulsePoint\\n4.4</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>51 to 200 employees</td>\n",
       "      <td>2011</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Internet</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>$100 to $500 million (USD)</td>\n",
       "      <td>Crossix Solutions Inc., AppNexus, The Trade Desk</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Data Scientist, Acorn AI Labs</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>Medidata: Conquering Diseases Together\\n\\nMedi...</td>\n",
       "      <td>4.3</td>\n",
       "      <td>Medidata Solutions\\n4.3</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>1001 to 5000 employees</td>\n",
       "      <td>1999</td>\n",
       "      <td>Company - Public</td>\n",
       "      <td>Enterprise Software &amp; Network Solutions</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>$500 million to $1 billion (USD)</td>\n",
       "      <td>Oracle</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>A Career with Point72’s MI Data team\\nAt Point...</td>\n",
       "      <td>3.9</td>\n",
       "      <td>Point72\\n3.9</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>Stamford, CT</td>\n",
       "      <td>1001 to 5000 employees</td>\n",
       "      <td>2014</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Investment Banking &amp; Asset Management</td>\n",
       "      <td>Finance</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Data Scientist - Alpha Insights</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>Two Sigma is a different kind of investment ma...</td>\n",
       "      <td>4.4</td>\n",
       "      <td>Two Sigma\\n4.4</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>1001 to 5000 employees</td>\n",
       "      <td>2001</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Investment Banking &amp; Asset Management</td>\n",
       "      <td>Finance</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>Data Scientist\\nAffinity Solutions / Marketing...</td>\n",
       "      <td>3.0</td>\n",
       "      <td>Affinity Solutions\\n3.0</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>51 to 200 employees</td>\n",
       "      <td>1998</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Advertising &amp; Marketing</td>\n",
       "      <td>Business Services</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>Commerce Signals, Cardlytics, Yodlee</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Data Scientist, Analytics</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>Company DescriptionAs an Etsy employee, you ca...</td>\n",
       "      <td>3.6</td>\n",
       "      <td>Etsy\\n3.6</td>\n",
       "      <td>Brooklyn, NY</td>\n",
       "      <td>Brooklyn, NY</td>\n",
       "      <td>501 to 1000 employees</td>\n",
       "      <td>2005</td>\n",
       "      <td>Company - Public</td>\n",
       "      <td>Other Retail Stores</td>\n",
       "      <td>Retail</td>\n",
       "      <td>$100 to $500 million (USD)</td>\n",
       "      <td>Airbnb, Warby Parker, Kickstarter</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Data Scientist/ML Engineer</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>Data Scientist/ML Engineer\\n\\nApply Now\\n\\nBec...</td>\n",
       "      <td>3.3</td>\n",
       "      <td>PA Consulting\\n3.3</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>London, United Kingdom</td>\n",
       "      <td>1001 to 5000 employees</td>\n",
       "      <td>1943</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Consulting</td>\n",
       "      <td>Business Services</td>\n",
       "      <td>$100 to $500 million (USD)</td>\n",
       "      <td>McKinsey &amp; Company, Accenture, Deloitte</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>Job Description\\nCompany Description\\n\\nAs an ...</td>\n",
       "      <td>3.6</td>\n",
       "      <td>Etsy\\n3.6</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>Brooklyn, NY</td>\n",
       "      <td>501 to 1000 employees</td>\n",
       "      <td>2005</td>\n",
       "      <td>Company - Public</td>\n",
       "      <td>Other Retail Stores</td>\n",
       "      <td>Retail</td>\n",
       "      <td>$100 to $500 million (USD)</td>\n",
       "      <td>Airbnb, Warby Parker, Kickstarter</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>VP, Data Science</td>\n",
       "      <td>$111K-$181K (Glassdoor est.)</td>\n",
       "      <td>We are looking for a VP, Data Science to lead ...</td>\n",
       "      <td>3.9</td>\n",
       "      <td>7Park Data\\n3.9</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>51 to 200 employees</td>\n",
       "      <td>2012</td>\n",
       "      <td>Company - Private</td>\n",
       "      <td>Research &amp; Development</td>\n",
       "      <td>Business Services</td>\n",
       "      <td>Unknown / Non-Applicable</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               Job Title               Salary Estimate  \\\n",
       "index                                                                    \n",
       "0                  Senior Data Scientist  $111K-$181K (Glassdoor est.)   \n",
       "1      Data Scientist, Product Analytics  $111K-$181K (Glassdoor est.)   \n",
       "2                   Data Science Manager  $111K-$181K (Glassdoor est.)   \n",
       "3                           Data Analyst  $111K-$181K (Glassdoor est.)   \n",
       "4                 Director, Data Science  $111K-$181K (Glassdoor est.)   \n",
       "5                         Data Scientist  $111K-$181K (Glassdoor est.)   \n",
       "6                Quantitative Researcher  $111K-$181K (Glassdoor est.)   \n",
       "7        Quantitative Research Associate  $111K-$181K (Glassdoor est.)   \n",
       "8                           AI Scientist  $111K-$181K (Glassdoor est.)   \n",
       "9                Quantitative Researcher  $111K-$181K (Glassdoor est.)   \n",
       "10                        Data Scientist  $111K-$181K (Glassdoor est.)   \n",
       "11       Data Scientist/Machine Learning  $111K-$181K (Glassdoor est.)   \n",
       "12         Data Scientist, Acorn AI Labs  $111K-$181K (Glassdoor est.)   \n",
       "13                        Data Scientist  $111K-$181K (Glassdoor est.)   \n",
       "14       Data Scientist - Alpha Insights  $111K-$181K (Glassdoor est.)   \n",
       "15                        Data Scientist  $111K-$181K (Glassdoor est.)   \n",
       "16             Data Scientist, Analytics  $111K-$181K (Glassdoor est.)   \n",
       "17            Data Scientist/ML Engineer  $111K-$181K (Glassdoor est.)   \n",
       "18                        Data Scientist  $111K-$181K (Glassdoor est.)   \n",
       "19                      VP, Data Science  $111K-$181K (Glassdoor est.)   \n",
       "\n",
       "                                         Job Description  Rating  \\\n",
       "index                                                              \n",
       "0      ABOUT HOPPER\\n\\nAt Hopper, we’re on a mission ...     3.5   \n",
       "1      At Noom, we use scientifically proven methods ...     4.5   \n",
       "2      Decode_M\\n\\nhttps://www.decode-m.com/\\n\\nData ...    -1.0   \n",
       "3      Sapphire Digital seeks a dynamic and driven mi...     3.4   \n",
       "4      Director, Data Science - (200537)\\nDescription...     3.4   \n",
       "5      Job Brief\\n\\nThe ideal candidate will have pre...     2.9   \n",
       "6      Experience: Entry-level (PhD Program) or Exper...     4.4   \n",
       "7      Seeking a quant to work with senior researcher...    -1.0   \n",
       "8      Paige is a software company helping pathologis...     5.0   \n",
       "9      About the Position\\n\\n\\nAt Jane Street, we con...     4.8   \n",
       "10     Company Description:\\n\\nQuartet is a pioneerin...     3.9   \n",
       "11     PulsePoint™, a global programmatic advertising...     4.4   \n",
       "12     Medidata: Conquering Diseases Together\\n\\nMedi...     4.3   \n",
       "13     A Career with Point72’s MI Data team\\nAt Point...     3.9   \n",
       "14     Two Sigma is a different kind of investment ma...     4.4   \n",
       "15     Data Scientist\\nAffinity Solutions / Marketing...     3.0   \n",
       "16     Company DescriptionAs an Etsy employee, you ca...     3.6   \n",
       "17     Data Scientist/ML Engineer\\n\\nApply Now\\n\\nBec...     3.3   \n",
       "18     Job Description\\nCompany Description\\n\\nAs an ...     3.6   \n",
       "19     We are looking for a VP, Data Science to lead ...     3.9   \n",
       "\n",
       "                          Company Name       Location            Headquarters  \\\n",
       "index                                                                           \n",
       "0                          Hopper\\n3.5   New York, NY        Montreal, Canada   \n",
       "1                         Noom US\\n4.5   New York, NY            New York, NY   \n",
       "2                             Decode_M   New York, NY            New York, NY   \n",
       "3                Sapphire Digital\\n3.4  Lyndhurst, NJ           Lyndhurst, NJ   \n",
       "4      United Entertainment Group\\n3.4   New York, NY            New York, NY   \n",
       "5                   IFG Companies\\n2.9   New York, NY            Hartford, CT   \n",
       "6                    PDT Partners\\n4.4   New York, NY            New York, NY   \n",
       "7               Enlightenment Research   New York, NY            New York, NY   \n",
       "8                           Paige\\n5.0   New York, NY            New York, NY   \n",
       "9                     Jane Street\\n4.8   New York, NY            New York, NY   \n",
       "10                 Quartet Health\\n3.9   New York, NY            New York, NY   \n",
       "11                     PulsePoint\\n4.4   New York, NY            New York, NY   \n",
       "12             Medidata Solutions\\n4.3   New York, NY            New York, NY   \n",
       "13                        Point72\\n3.9   New York, NY            Stamford, CT   \n",
       "14                      Two Sigma\\n4.4   New York, NY            New York, NY   \n",
       "15             Affinity Solutions\\n3.0   New York, NY            New York, NY   \n",
       "16                           Etsy\\n3.6   Brooklyn, NY            Brooklyn, NY   \n",
       "17                  PA Consulting\\n3.3   New York, NY  London, United Kingdom   \n",
       "18                           Etsy\\n3.6   New York, NY            Brooklyn, NY   \n",
       "19                     7Park Data\\n3.9   New York, NY            New York, NY   \n",
       "\n",
       "                         Size  Founded  Type of ownership  \\\n",
       "index                                                       \n",
       "0       501 to 1000 employees     2007  Company - Private   \n",
       "1      1001 to 5000 employees     2008  Company - Private   \n",
       "2           1 to 50 employees       -1            Unknown   \n",
       "3        201 to 500 employees     2019  Company - Private   \n",
       "4         51 to 200 employees     2007  Company - Private   \n",
       "5        201 to 500 employees     1985  Company - Private   \n",
       "6         51 to 200 employees     1993  Company - Private   \n",
       "7           1 to 50 employees       -1            Unknown   \n",
       "8           1 to 50 employees     2018  Company - Private   \n",
       "9       501 to 1000 employees     2000  Company - Private   \n",
       "10       201 to 500 employees     2014  Company - Private   \n",
       "11        51 to 200 employees     2011  Company - Private   \n",
       "12     1001 to 5000 employees     1999   Company - Public   \n",
       "13     1001 to 5000 employees     2014  Company - Private   \n",
       "14     1001 to 5000 employees     2001  Company - Private   \n",
       "15        51 to 200 employees     1998  Company - Private   \n",
       "16      501 to 1000 employees     2005   Company - Public   \n",
       "17     1001 to 5000 employees     1943  Company - Private   \n",
       "18      501 to 1000 employees     2005   Company - Public   \n",
       "19        51 to 200 employees     2012  Company - Private   \n",
       "\n",
       "                                      Industry                  Sector  \\\n",
       "index                                                                    \n",
       "0                              Travel Agencies        Travel & Tourism   \n",
       "1                    Health, Beauty, & Fitness       Consumer Services   \n",
       "2                                           -1                      -1   \n",
       "3                                     Internet  Information Technology   \n",
       "4                      Advertising & Marketing       Business Services   \n",
       "5                           Insurance Carriers               Insurance   \n",
       "6        Investment Banking & Asset Management                 Finance   \n",
       "7                                           -1                      -1   \n",
       "8      Enterprise Software & Network Solutions  Information Technology   \n",
       "9        Investment Banking & Asset Management                 Finance   \n",
       "10     Enterprise Software & Network Solutions  Information Technology   \n",
       "11                                    Internet  Information Technology   \n",
       "12     Enterprise Software & Network Solutions  Information Technology   \n",
       "13       Investment Banking & Asset Management                 Finance   \n",
       "14       Investment Banking & Asset Management                 Finance   \n",
       "15                     Advertising & Marketing       Business Services   \n",
       "16                         Other Retail Stores                  Retail   \n",
       "17                                  Consulting       Business Services   \n",
       "18                         Other Retail Stores                  Retail   \n",
       "19                      Research & Development       Business Services   \n",
       "\n",
       "                                Revenue  \\\n",
       "index                                     \n",
       "0              Unknown / Non-Applicable   \n",
       "1              Unknown / Non-Applicable   \n",
       "2              Unknown / Non-Applicable   \n",
       "3              Unknown / Non-Applicable   \n",
       "4              Unknown / Non-Applicable   \n",
       "5              Unknown / Non-Applicable   \n",
       "6              Unknown / Non-Applicable   \n",
       "7              Unknown / Non-Applicable   \n",
       "8              Unknown / Non-Applicable   \n",
       "9              Unknown / Non-Applicable   \n",
       "10             Unknown / Non-Applicable   \n",
       "11           $100 to $500 million (USD)   \n",
       "12     $500 million to $1 billion (USD)   \n",
       "13             Unknown / Non-Applicable   \n",
       "14             Unknown / Non-Applicable   \n",
       "15             Unknown / Non-Applicable   \n",
       "16           $100 to $500 million (USD)   \n",
       "17           $100 to $500 million (USD)   \n",
       "18           $100 to $500 million (USD)   \n",
       "19             Unknown / Non-Applicable   \n",
       "\n",
       "                                            Competitors Easy Apply  \n",
       "index                                                               \n",
       "0                                                    -1         -1  \n",
       "1                                                    -1         -1  \n",
       "2                                                    -1       True  \n",
       "3                                  Zocdoc, Healthgrades         -1  \n",
       "4                              BBDO, Grey Group, Droga5         -1  \n",
       "5                         Colony Specialty, Markel, RLI         -1  \n",
       "6                                                    -1         -1  \n",
       "7                                                    -1       True  \n",
       "8                                                    -1       True  \n",
       "9                                                    -1         -1  \n",
       "10                                                   -1       True  \n",
       "11     Crossix Solutions Inc., AppNexus, The Trade Desk         -1  \n",
       "12                                               Oracle         -1  \n",
       "13                                                   -1         -1  \n",
       "14                                                   -1         -1  \n",
       "15                 Commerce Signals, Cardlytics, Yodlee       True  \n",
       "16                    Airbnb, Warby Parker, Kickstarter         -1  \n",
       "17              McKinsey & Company, Accenture, Deloitte         -1  \n",
       "18                    Airbnb, Warby Parker, Kickstarter         -1  \n",
       "19                                                   -1         -1  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "e96d3001-7ac3-4bf7-8ac5-3e971fd8da90",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 3909 entries, 0 to 4379\n",
      "Data columns (total 15 columns):\n",
      " #   Column             Non-Null Count  Dtype  \n",
      "---  ------             --------------  -----  \n",
      " 0   Job Title          3909 non-null   object \n",
      " 1   Salary Estimate    3909 non-null   object \n",
      " 2   Job Description    3909 non-null   object \n",
      " 3   Rating             3909 non-null   float64\n",
      " 4   Company Name       3909 non-null   object \n",
      " 5   Location           3909 non-null   object \n",
      " 6   Headquarters       3909 non-null   object \n",
      " 7   Size               3909 non-null   object \n",
      " 8   Founded            3909 non-null   int64  \n",
      " 9   Type of ownership  3909 non-null   object \n",
      " 10  Industry           3909 non-null   object \n",
      " 11  Sector             3909 non-null   object \n",
      " 12  Revenue            3909 non-null   object \n",
      " 13  Competitors        3909 non-null   object \n",
      " 14  Easy Apply         3909 non-null   object \n",
      "dtypes: float64(1), int64(1), object(13)\n",
      "memory usage: 488.6+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8a4ae2d-e262-44c8-ae6a-cc46d7b22dd4",
   "metadata": {},
   "source": [
    "### Job Title: Categorizing  job title into different field of career\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "15ac6f6e-d212-4896-abba-260591d0a891",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Job Title', 'Salary Estimate', 'Job Description', 'Rating',\n",
       "       'Company Name', 'Location', 'Headquarters', 'Size', 'Founded',\n",
       "       'Type of ownership', 'Industry', 'Sector', 'Revenue', 'Competitors',\n",
       "       'Easy Apply'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "053a46bb-1b63-4134-8f0a-4899ca5142dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Created a function to iterate through the Series for data scientist  job listing title\n",
    "\n",
    "def job_cat(title):\n",
    "    if 'data scientist' in title.lower():\n",
    "        return 'data scientist'\n",
    "    elif 'data engineer' in title.lower():\n",
    "        return 'Data Engineer'\n",
    "    elif 'data analyst' in title.lower():\n",
    "        return 'data analyst'\n",
    "    elif 'machine learning' in title.lower():\n",
    "        return 'machine learning'\n",
    "    elif 'business' in title.lower():\n",
    "        return 'business i/da'\n",
    "    else:\n",
    "        return 'na'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "a7bda4f8-6675-44b4-a26f-05c2dcc8b218",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Created a function to iterate through the Series for level of job listing title\n",
    "\n",
    "def seniority(level):\n",
    "    if 'senior' in level.lower() or 'sr' in level.lower() or 'lead' in level.lower() or 'principal' in level.lower() or 'manager' in level.lower():\n",
    "        return 'senior'\n",
    "    elif 'junior' in level.lower():\n",
    "        return 'junior'\n",
    "    else:\n",
    "        return 'na'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "5e4a1d45-14ce-4cdd-8cc0-9480447cb438",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['job_cat'] = df['Job Title'].apply(job_cat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "9fd0c5b7-3d42-460e-b5e1-ad28fe79d9a1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "na                  1201\n",
       "data scientist       984\n",
       "data analyst         747\n",
       "Data Engineer        742\n",
       "machine learning     140\n",
       "business i/da         95\n",
       "Name: job_cat, dtype: int64"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['job_cat'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "178802c6-e595-4243-9fb3-35c03946dd13",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['entry_level'] = df['Job Title'].apply(seniority)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "03983583-f40c-432e-a2ad-72891be77899",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "na        2881\n",
       "senior     993\n",
       "junior      35\n",
       "Name: entry_level, dtype: int64"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['entry_level'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85247009-cc5e-4140-b344-2ddea08dd13c",
   "metadata": {},
   "source": [
    "### Salary Estimate: parsing (Min, Max, Hourly)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "1ce1f4cf-818d-4553-babd-9849287475ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index\n",
       "0    $111K-$181K (Glassdoor est.)\n",
       "1    $111K-$181K (Glassdoor est.)\n",
       "2    $111K-$181K (Glassdoor est.)\n",
       "3    $111K-$181K (Glassdoor est.)\n",
       "4    $111K-$181K (Glassdoor est.)\n",
       "Name: Salary Estimate, dtype: object"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Salary Estimate'].head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "de81eaae-8148-4ac9-a387-28bfb1684933",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Hourly Salary \n",
    "\n",
    "df['hourly'] = df['Salary Estimate'].apply(lambda x: 1 if 'per hour' in x.lower() else 0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "07156278-ced0-4ecf-94e9-b7ee282cdb4a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    3888\n",
       "1      21\n",
       "Name: hourly, dtype: int64"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['hourly'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "058db79a-a6e3-48d4-b38d-41ecc6469459",
   "metadata": {},
   "outputs": [],
   "source": [
    "# removing (Glassdoor est.), $ and K \n",
    "\n",
    "salary_est = df['Salary Estimate'].apply(lambda x: x.split('(')[0].replace('$', '').replace('K', ''))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "0333308e-7a58-4d21-88cd-198b3d325242",
   "metadata": {},
   "outputs": [],
   "source": [
    "salary_est = salary_est.apply(lambda x: x.lower().replace('per hour', '').strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "06dfec71-dc09-43e6-9db8-2b33204de5fe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index\n",
       "0    111-181\n",
       "1    111-181\n",
       "2    111-181\n",
       "3    111-181\n",
       "4    111-181\n",
       "Name: Salary Estimate, dtype: object"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "salary_est.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "b7fc93e6-5421-41db-b97f-7a1e6a965303",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Minimum Salary\n",
    "\n",
    "df['min_salary'] = salary_est.apply(lambda x: x.split('-')[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "df0a0ce0-e28c-45b4-a34b-33f900f868e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index\n",
       "0    111\n",
       "1    111\n",
       "2    111\n",
       "Name: min_salary, dtype: int32"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Conver the Series to integer\n",
    "\n",
    "df['min_salary'] = df['min_salary'].astype('int')\n",
    "df['min_salary'].head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "3d185eec-0d1b-490f-8822-9a0a76d655f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Maximum Salary\n",
    "\n",
    "df['max_salary'] = salary_est.apply(lambda x: x.split('-')[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "e196ec75-1c7e-4e41-8da9-bd5fcf9eddd4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index\n",
       "0    181\n",
       "1    181\n",
       "2    181\n",
       "Name: max_salary, dtype: int32"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Conver the Series to integer\n",
    "\n",
    "df['max_salary'] = df['max_salary'].astype('int')\n",
    "df['max_salary'].head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "13cc43ce-663a-4d18-ac7e-baa9b8ea0b1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Average Salary\n",
    "\n",
    "df['Avg_salary'] = (df['min_salary'] + df['max_salary']) / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "99c27d07-4547-45f2-9e81-26b830a01a4e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index\n",
       "0    146.0\n",
       "1    146.0\n",
       "2    146.0\n",
       "3    146.0\n",
       "4    146.0\n",
       "Name: Avg_salary, dtype: float64"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Avg_salary'].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "99322ecc-7472-4ca9-a516-cfed0cdff1f3",
   "metadata": {},
   "source": [
    "### Job Desc.: parsing (python, R, Spark, Aws, Excel, Sql, etc..)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "850d3ead-41a1-4c98-af33-faafe52a5a4d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Job Title', 'Salary Estimate', 'Job Description', 'Rating',\n",
       "       'Company Name', 'Location', 'Headquarters', 'Size', 'Founded',\n",
       "       'Type of ownership', 'Industry', 'Sector', 'Revenue', 'Competitors',\n",
       "       'Easy Apply', 'job_cat', 'entry_level', 'hourly', 'min_salary',\n",
       "       'max_salary', 'Avg_salary'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aaea1442-3af2-4c15-a090-068dc38c68f9",
   "metadata": {},
   "source": [
    "    Skill/Tools required that is found in the job listing\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "ef3f4a00-b378-4ba4-b647-0e00b9a68803",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Python\n",
    "df['Python_req'] = df['Job Description'].apply(lambda x: 1 if 'python' in x.lower() else 0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "cfce972b-3f50-48a1-81f4-49075560ed78",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    2057\n",
       "0    1852\n",
       "Name: Python_req, dtype: int64"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Python_req'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "ed2c4930-2231-4df5-99b5-f3d94524e0c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# R Studio\n",
    "df['RStudio_req'] = df['Job Description'].apply(lambda x: 1 if 'r studio' in x.lower() or 'r-studio' in x.lower() else 0 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "1fc8092d-2817-4ed0-9014-4232d07d154b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    3891\n",
       "1      18\n",
       "Name: RStudio_req, dtype: int64"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['RStudio_req'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "c9d7838d-4148-4527-8bbc-9599040d0548",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saprk\n",
    "df['Spark_req'] = df['Job Description'].apply(lambda x: 1 if 'spark' in x.lower() or 'apache spark' in x.lower() else 0 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "a267908f-77b4-41cb-b72d-2b29cef54cf0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    3012\n",
       "1     897\n",
       "Name: Spark_req, dtype: int64"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Spark_req'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "6aa5c497-4ae3-472c-a035-999389f8eaf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# AWS\n",
    "df['Aws_req'] = df['Job Description'].apply(lambda x: 1 if 'aws' in x.lower() else 0 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "9013c8d2-c6d4-42d5-8d7e-d2358a57397f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    2960\n",
       "1     949\n",
       "Name: Aws_req, dtype: int64"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Aws_req'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "362b5894-cd00-4f78-af95-b69a0621873e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Excel\n",
    "df['Excel_req'] = df['Job Description'].apply(lambda x: 1 if 'excel' in x.lower() else 0 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "ba61f658-ffb0-490d-9d16-d774d4d17838",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    2054\n",
       "1    1855\n",
       "Name: Excel_req, dtype: int64"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Excel_req'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63d6b30b-130f-4477-8eb3-246cc948e137",
   "metadata": {},
   "source": [
    "### Rating: replace -1 with 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "d874e099-f533-4e4b-8345-657dcd88929f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index\n",
       "0    3.5\n",
       "1    4.5\n",
       "2   -1.0\n",
       "3    3.4\n",
       "4    3.4\n",
       "5    2.9\n",
       "6    4.4\n",
       "7   -1.0\n",
       "8    5.0\n",
       "9    4.8\n",
       "Name: Rating, dtype: float64"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Rating'].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "2aa3de50-edfa-47d8-b60d-97d4f85f182c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Rating'].replace(-1.0, 0.0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "8c78541b-4d5a-4ba4-8010-a6d74004c335",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index\n",
       "0    3.5\n",
       "1    4.5\n",
       "2    0.0\n",
       "3    3.4\n",
       "4    3.4\n",
       "Name: Rating, dtype: float64"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Rating'].head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "31657aaa-e769-4c88-a07c-2fbf4cb33d9e",
   "metadata": {},
   "source": [
    "### Company Name: stripping the rating values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "a643422c-26e0-468e-a8d4-fd67eed8755d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Job Title', 'Salary Estimate', 'Job Description', 'Rating',\n",
       "       'Company Name', 'Location', 'Headquarters', 'Size', 'Founded',\n",
       "       'Type of ownership', 'Industry', 'Sector', 'Revenue', 'Competitors',\n",
       "       'Easy Apply', 'job_cat', 'entry_level', 'hourly', 'min_salary',\n",
       "       'max_salary', 'Avg_salary', 'Python_req', 'RStudio_req', 'Spark_req',\n",
       "       'Aws_req', 'Excel_req'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "5584e833-0807-47d4-904a-67ab078d2809",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index\n",
       "0                        Hopper\\n3.5\n",
       "1                       Noom US\\n4.5\n",
       "2                           Decode_M\n",
       "3              Sapphire Digital\\n3.4\n",
       "4    United Entertainment Group\\n3.4\n",
       "5                 IFG Companies\\n2.9\n",
       "6                  PDT Partners\\n4.4\n",
       "7             Enlightenment Research\n",
       "8                         Paige\\n5.0\n",
       "9                   Jane Street\\n4.8\n",
       "Name: Company Name, dtype: object"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Company Name'].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "f5b1fbda-41c9-422a-ac42-f6c2b669e195",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Company_nm'] = df.apply(lambda x: x['Company Name'] if x['Rating'] == 0 else x['Company Name'][:-4], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "80413c7c-e5e3-4842-9d84-274562761dec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index\n",
       "0                        Hopper\n",
       "1                       Noom US\n",
       "2                      Decode_M\n",
       "3              Sapphire Digital\n",
       "4    United Entertainment Group\n",
       "5                 IFG Companies\n",
       "6                  PDT Partners\n",
       "7        Enlightenment Research\n",
       "8                         Paige\n",
       "9                   Jane Street\n",
       "Name: Company_nm, dtype: object"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Company_nm'].head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "70afe85a-f09b-4796-a31c-a0ae73d67b0e",
   "metadata": {},
   "source": [
    "### Location: parsing state code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "cfcbecae-3478-4a2b-b8ad-62fa1ab58dcd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Job Title', 'Salary Estimate', 'Job Description', 'Rating',\n",
       "       'Company Name', 'Location', 'Headquarters', 'Size', 'Founded',\n",
       "       'Type of ownership', 'Industry', 'Sector', 'Revenue', 'Competitors',\n",
       "       'Easy Apply', 'job_cat', 'entry_level', 'hourly', 'min_salary',\n",
       "       'max_salary', 'Avg_salary', 'Python_req', 'RStudio_req', 'Spark_req',\n",
       "       'Aws_req', 'Excel_req', 'Company_nm'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "17efc345-1a1a-4c5d-b308-3b9113f41173",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index\n",
       "0     New York, NY\n",
       "1     New York, NY\n",
       "2     New York, NY\n",
       "3    Lyndhurst, NJ\n",
       "4     New York, NY\n",
       "Name: Location, dtype: object"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Location'].head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "8aa9c17d-c1be-4e3f-8d0d-acee2566c178",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Job_state'] = df['Location'].apply(lambda x: x.split(',')[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "c3eafeab-abf9-4a58-997b-c8aee542ecfd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index\n",
       "0     NY\n",
       "1     NY\n",
       "2     NY\n",
       "3     NJ\n",
       "4     NY\n",
       "Name: Job_state, dtype: object"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Job_state'].head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "5ceb7d2b-199a-43f9-91e9-75342429daff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       " TX                1253\n",
       " CA                1069\n",
       " IL                 363\n",
       " PA                 324\n",
       " NY                 313\n",
       " AZ                 295\n",
       " OH                 177\n",
       " FL                  69\n",
       " NJ                  32\n",
       " DE                  10\n",
       " United Kingdom       4\n",
       "Name: Job_state, dtype: int64"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Job_state'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "5768b563-3745-4694-8ada-714d264dff3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Job_state'] = df['Job_state'].apply(lambda x: x.replace('United Kingdom', 'UK'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "c89cb7b1-d8b6-4074-96d0-a99cf80324d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       " TX    1253\n",
       " CA    1069\n",
       " IL     363\n",
       " PA     324\n",
       " NY     313\n",
       " AZ     295\n",
       " OH     177\n",
       " FL      69\n",
       " NJ      32\n",
       " DE      10\n",
       " UK       4\n",
       "Name: Job_state, dtype: int64"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Job_state'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6a376b75-3e22-489d-9de3-e66bc060afc6",
   "metadata": {},
   "source": [
    "#### Headquarter: check if job is in same location or in HQ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "30acbecb-ac80-40c9-af55-70937eef392e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Job Title', 'Salary Estimate', 'Job Description', 'Rating',\n",
       "       'Company Name', 'Location', 'Headquarters', 'Size', 'Founded',\n",
       "       'Type of ownership', 'Industry', 'Sector', 'Revenue', 'Competitors',\n",
       "       'Easy Apply', 'job_cat', 'entry_level', 'hourly', 'min_salary',\n",
       "       'max_salary', 'Avg_salary', 'Python_req', 'RStudio_req', 'Spark_req',\n",
       "       'Aws_req', 'Excel_req', 'Company_nm', 'Job_state'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "269acfd3-a48b-40d8-b3fd-8e0c4e434cd4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index\n",
       "0           Montreal, Canada\n",
       "1               New York, NY\n",
       "2               New York, NY\n",
       "3              Lyndhurst, NJ\n",
       "4               New York, NY\n",
       "5               Hartford, CT\n",
       "6               New York, NY\n",
       "7               New York, NY\n",
       "8               New York, NY\n",
       "9               New York, NY\n",
       "10              New York, NY\n",
       "11              New York, NY\n",
       "12              New York, NY\n",
       "13              Stamford, CT\n",
       "14              New York, NY\n",
       "15              New York, NY\n",
       "16              Brooklyn, NY\n",
       "17    London, United Kingdom\n",
       "18              Brooklyn, NY\n",
       "19              New York, NY\n",
       "Name: Headquarters, dtype: object"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Headquarters'].head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "388fcdfb-e934-49c0-9418-91c5727a7b12",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Same_state'] = df.apply(lambda x: 1 if x['Location'] in x['Headquarters'] else 0, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "654491fc-b888-41a1-b76b-34d14d5e9b52",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    2688\n",
       "1    1221\n",
       "Name: Same_state, dtype: int64"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Same_state'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "96a074d7-7102-4d17-a61a-8bdee9269522",
   "metadata": {},
   "source": [
    "#### Size: Parsing (Min, Max, Na) strip -1 with 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "63d05078-7e6f-453f-b970-7c448b9ef5b3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10000+ employees           993\n",
       "51 to 200 employees        563\n",
       "1001 to 5000 employees     553\n",
       "1 to 50 employees          550\n",
       "201 to 500 employees       418\n",
       "501 to 1000 employees      307\n",
       "-1                         229\n",
       "5001 to 10000 employees    219\n",
       "Unknown                     77\n",
       "Name: Size, dtype: int64"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Size'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "ee906163-c5a2-4112-bc24-389ced4bbbc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Size'].replace({'-1': 'Na', 'Unknown': 'Na' }, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "03d0ab0f-58bb-4a79-a52c-41927c9e2b75",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10000+ employees           993\n",
       "51 to 200 employees        563\n",
       "1001 to 5000 employees     553\n",
       "1 to 50 employees          550\n",
       "201 to 500 employees       418\n",
       "501 to 1000 employees      307\n",
       "Na                         306\n",
       "5001 to 10000 employees    219\n",
       "Name: Size, dtype: int64"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Size'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fabd725d-389f-469b-8489-10fd40ec0a7a",
   "metadata": {},
   "source": [
    "#### Founded: \n",
    "Get the age of company while stripping -1 with 0. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "21ce8bdf-9319-4ae9-9261-3200e75c037f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index\n",
       "0    2007\n",
       "1    2008\n",
       "2      -1\n",
       "3    2019\n",
       "4    2007\n",
       "5    1985\n",
       "6    1993\n",
       "7      -1\n",
       "8    2018\n",
       "9    2000\n",
       "Name: Founded, dtype: int64"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Founded'].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "b93a9b96-c8fd-45da-9458-2f1de5584210",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Founded'].replace(-1, 0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "febb9434-de7f-46c6-af70-eb1946a4fc0c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index\n",
       "0    2007\n",
       "1    2008\n",
       "2       0\n",
       "3    2019\n",
       "4    2007\n",
       "5    1985\n",
       "6    1993\n",
       "7       0\n",
       "8    2018\n",
       "9    2000\n",
       "Name: Founded, dtype: int64"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Founded'].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "341d00e2-28cf-423e-90a6-9b8e0867d090",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Age'] = df['Founded'].apply(lambda x: x if x == 0 else 2022 - x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "d72e5de1-6a97-41d2-9e3d-dcbd13f465cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index\n",
       "0    15\n",
       "1    14\n",
       "2     0\n",
       "3     3\n",
       "4    15\n",
       "5    37\n",
       "6    29\n",
       "7     0\n",
       "8     4\n",
       "9    22\n",
       "Name: Age, dtype: int64"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Age'].head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f7cf2d19-0836-4b97-b361-d827c68dea63",
   "metadata": {},
   "source": [
    "#### Type of ownership: categorize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "9064c6ed-b1f6-4ecf-a107-d38a6a01f018",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Job Title', 'Salary Estimate', 'Job Description', 'Rating',\n",
       "       'Company Name', 'Location', 'Headquarters', 'Size', 'Founded',\n",
       "       'Type of ownership', 'Industry', 'Sector', 'Revenue', 'Competitors',\n",
       "       'Easy Apply', 'job_cat', 'entry_level', 'hourly', 'min_salary',\n",
       "       'max_salary', 'Avg_salary', 'Python_req', 'RStudio_req', 'Spark_req',\n",
       "       'Aws_req', 'Excel_req', 'Company_nm', 'Job_state', 'Same_state', 'Age'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "5e7d81da-db1c-4a60-a190-af2b7c4929a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Company - Private                 1933\n",
       "Company - Public                  1114\n",
       "-1                                 229\n",
       "Nonprofit Organization             207\n",
       "Subsidiary or Business Segment     160\n",
       "Government                          91\n",
       "College / University                77\n",
       "Unknown                             38\n",
       "Contract                            17\n",
       "Hospital                            16\n",
       "Private Practice / Firm             12\n",
       "School / School District             5\n",
       "Other Organization                   5\n",
       "Self-employed                        4\n",
       "Franchise                            1\n",
       "Name: Type of ownership, dtype: int64"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Type of ownership'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "6e310b8f-0913-4786-80ec-fb6c34327795",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Type of ownership'].replace({'-1': 'Na','Unknown': 'Na'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "cce4653d-9d3c-48e0-84f7-23feb2638cce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Company - Private                 1933\n",
       "Company - Public                  1114\n",
       "Na                                 267\n",
       "Nonprofit Organization             207\n",
       "Subsidiary or Business Segment     160\n",
       "Government                          91\n",
       "College / University                77\n",
       "Contract                            17\n",
       "Hospital                            16\n",
       "Private Practice / Firm             12\n",
       "School / School District             5\n",
       "Other Organization                   5\n",
       "Self-employed                        4\n",
       "Franchise                            1\n",
       "Name: Type of ownership, dtype: int64"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Type of ownership'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "4c89a710-65d8-402c-910a-ab63ee9a5657",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Job Title', 'Salary Estimate', 'Job Description', 'Rating',\n",
       "       'Company Name', 'Location', 'Headquarters', 'Size', 'Founded',\n",
       "       'Type of ownership', 'Industry', 'Sector', 'Revenue', 'Competitors',\n",
       "       'Easy Apply', 'job_cat', 'entry_level', 'hourly', 'min_salary',\n",
       "       'max_salary', 'Avg_salary', 'Python_req', 'RStudio_req', 'Spark_req',\n",
       "       'Aws_req', 'Excel_req', 'Company_nm', 'Job_state', 'Same_state', 'Age'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "182aff3b-1b39-4243-a739-71ccb6fba2a1",
   "metadata": {},
   "source": [
    "#### Industry: categorize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "f7e892af-3958-4e96-8dbc-2e9f5867d949",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index\n",
       "0                             Travel Agencies\n",
       "1                   Health, Beauty, & Fitness\n",
       "2                                          -1\n",
       "3                                    Internet\n",
       "4                     Advertising & Marketing\n",
       "5                          Insurance Carriers\n",
       "6       Investment Banking & Asset Management\n",
       "7                                          -1\n",
       "8     Enterprise Software & Network Solutions\n",
       "9       Investment Banking & Asset Management\n",
       "10    Enterprise Software & Network Solutions\n",
       "11                                   Internet\n",
       "12    Enterprise Software & Network Solutions\n",
       "13      Investment Banking & Asset Management\n",
       "14      Investment Banking & Asset Management\n",
       "15                    Advertising & Marketing\n",
       "16                        Other Retail Stores\n",
       "17                                 Consulting\n",
       "18                        Other Retail Stores\n",
       "19                     Research & Development\n",
       "Name: Industry, dtype: object"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Industry'].head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "dc85b78f-4229-4e05-9338-b2b6a94c34cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1                                          546\n",
       "IT Services                                 472\n",
       "Staffing & Outsourcing                      313\n",
       "Biotech & Pharmaceuticals                   293\n",
       "Computer Hardware & Software                264\n",
       "Internet                                    238\n",
       "Health Care Services & Hospitals            197\n",
       "Consulting                                  166\n",
       "Enterprise Software & Network Solutions     150\n",
       "Investment Banking & Asset Management       109\n",
       "Advertising & Marketing                      93\n",
       "Insurance Carriers                           91\n",
       "Colleges & Universities                      84\n",
       "Banks & Credit Unions                        64\n",
       "Accounting                                   64\n",
       "Research & Development                       60\n",
       "Federal Agencies                             57\n",
       "Lending                                      41\n",
       "Aerospace & Defense                          37\n",
       "Video Games                                  30\n",
       "Architectural & Engineering Services         25\n",
       "TV Broadcast & Cable Networks                23\n",
       "Electrical & Electronic Manufacturing        19\n",
       "Department, Clothing, & Shoe Stores          18\n",
       "Health, Beauty, & Fitness                    18\n",
       "Insurance Agencies & Brokerages              18\n",
       "Brokerage Services                           17\n",
       "Other Retail Stores                          17\n",
       "Financial Analytics & Research               17\n",
       "Financial Transaction Processing             16\n",
       "Health Care Products Manufacturing           16\n",
       "State & Regional Agencies                    15\n",
       "Construction                                 15\n",
       "Oil & Gas Services                           15\n",
       "Social Assistance                            14\n",
       "Food & Beverage Manufacturing                13\n",
       "Consumer Products Manufacturing              12\n",
       "Real Estate                                  12\n",
       "Motion Picture Production & Distribution     12\n",
       "Education Training Services                  12\n",
       "Transportation Equipment Manufacturing       11\n",
       "Energy                                       11\n",
       "Telecommunications Services                  11\n",
       "Wholesale                                    11\n",
       "Municipal Governments                        10\n",
       "Cable, Internet & Telephone Providers         9\n",
       "Industrial Manufacturing                      9\n",
       "Logistics & Supply Chain                      8\n",
       "Grocery Stores & Supermarkets                 8\n",
       "Chemical Manufacturing                        7\n",
       "Legal                                         7\n",
       "Automotive Parts & Accessories Stores         6\n",
       "Publishing                                    6\n",
       "Food & Beverage Stores                        5\n",
       "K-12 Education                                5\n",
       "Oil & Gas Exploration & Production            5\n",
       "Casual Restaurants                            5\n",
       "Sports & Recreation                           5\n",
       "Vehicle Dealers                               4\n",
       "Pet & Pet Supplies Stores                     4\n",
       "Consumer Electronics & Appliances Stores      4\n",
       "Membership Organizations                      4\n",
       "Stock Exchanges                               4\n",
       "Utilities                                     3\n",
       "Gas Stations                                  3\n",
       "Farm Support Services                         3\n",
       "Building & Personnel Services                 3\n",
       "Security Services                             3\n",
       "Trucking                                      2\n",
       "Gambling                                      2\n",
       "Audiovisual                                   2\n",
       "Express Delivery Services                     2\n",
       "Radio                                         2\n",
       "Convenience Stores & Truck Stops              2\n",
       "Fast-Food & Quick-Service Restaurants         2\n",
       "Travel Agencies                               2\n",
       "Cruise Ships                                  2\n",
       "News Outlet                                   2\n",
       "Grantmaking Foundations                       2\n",
       "Food Production                               2\n",
       "Transportation Management                     2\n",
       "Venture Capital & Private Equity              2\n",
       "Telecommunications Manufacturing              1\n",
       "Home Furniture & Housewares Stores            1\n",
       "Self-Storage Services                         1\n",
       "Religious Organizations                       1\n",
       "Shipping                                      1\n",
       "Health Fundraising Organizations              1\n",
       "Consumer Product Rental                       1\n",
       "Museums, Zoos & Amusement Parks               1\n",
       "Sporting Goods Stores                         1\n",
       "Commercial Equipment Rental                   1\n",
       "Metals Brokers                                1\n",
       "Truck Rental & Leasing                        1\n",
       "Beauty & Personal Accessories Stores          1\n",
       "Auto Repair & Maintenance                     1\n",
       "Name: Industry, dtype: int64"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Industry'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "403f1b2a-eec3-4e61-8356-3db4585465b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Industry'].replace('-1', 'Others', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "b020b8c8-c717-4364-b9de-a8abb0013afc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index\n",
       "0              Travel Agencies\n",
       "1    Health, Beauty, & Fitness\n",
       "2                       Others\n",
       "3                     Internet\n",
       "4      Advertising & Marketing\n",
       "Name: Industry, dtype: object"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Industry'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "5a85d9fb-bda6-4771-af43-eaf33ad70150",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index\n",
       "0          Travel & Tourism\n",
       "1         Consumer Services\n",
       "2                        -1\n",
       "3    Information Technology\n",
       "4         Business Services\n",
       "5                 Insurance\n",
       "6                   Finance\n",
       "7                        -1\n",
       "8    Information Technology\n",
       "9                   Finance\n",
       "Name: Sector, dtype: object"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Sector'].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "a244a9fd-e996-4e86-afad-9edb8e49c64c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Information Technology                1124\n",
       "Business Services                      679\n",
       "-1                                     546\n",
       "Biotech & Pharmaceuticals              293\n",
       "Finance                                270\n",
       "Health Care                            197\n",
       "Insurance                              109\n",
       "Education                              101\n",
       "Manufacturing                           87\n",
       "Government                              82\n",
       "Media                                   75\n",
       "Retail                                  72\n",
       "Accounting & Legal                      71\n",
       "Aerospace & Defense                     37\n",
       "Oil, Gas, Energy & Utilities            34\n",
       "Telecommunications                      21\n",
       "Consumer Services                       20\n",
       "Non-Profit                              18\n",
       "Construction, Repair & Maintenance      16\n",
       "Transportation & Logistics              16\n",
       "Real Estate                             12\n",
       "Arts, Entertainment & Recreation        10\n",
       "Restaurants, Bars & Food Services        9\n",
       "Agriculture & Forestry                   5\n",
       "Travel & Tourism                         4\n",
       "Mining & Metals                          1\n",
       "Name: Sector, dtype: int64"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Sector'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "3efe4cad-1d1a-47a1-b954-ad09dd74eb8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Sector'].replace('-1', 'Others', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "583ea2b0-57e1-49ad-9fd9-c83b38c9fdb6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index\n",
       "0          Travel & Tourism\n",
       "1         Consumer Services\n",
       "2                    Others\n",
       "3    Information Technology\n",
       "4         Business Services\n",
       "5                 Insurance\n",
       "6                   Finance\n",
       "7                    Others\n",
       "8    Information Technology\n",
       "9                   Finance\n",
       "Name: Sector, dtype: object"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Sector'].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "07f628ee-c78d-41a8-aa2d-9882af100f29",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index\n",
       "0             Unknown / Non-Applicable\n",
       "1             Unknown / Non-Applicable\n",
       "2             Unknown / Non-Applicable\n",
       "3             Unknown / Non-Applicable\n",
       "4             Unknown / Non-Applicable\n",
       "5             Unknown / Non-Applicable\n",
       "6             Unknown / Non-Applicable\n",
       "7             Unknown / Non-Applicable\n",
       "8             Unknown / Non-Applicable\n",
       "9             Unknown / Non-Applicable\n",
       "10            Unknown / Non-Applicable\n",
       "11          $100 to $500 million (USD)\n",
       "12    $500 million to $1 billion (USD)\n",
       "13            Unknown / Non-Applicable\n",
       "14            Unknown / Non-Applicable\n",
       "15            Unknown / Non-Applicable\n",
       "16          $100 to $500 million (USD)\n",
       "17          $100 to $500 million (USD)\n",
       "18          $100 to $500 million (USD)\n",
       "19            Unknown / Non-Applicable\n",
       "Name: Revenue, dtype: object"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Revenue'].head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "d5596292-f86a-4916-835c-fff14bc5c936",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Unknown / Non-Applicable            1163\n",
       "$10+ billion (USD)                   599\n",
       "$100 to $500 million (USD)           338\n",
       "-1                                   229\n",
       "$50 to $100 million (USD)            214\n",
       "$2 to $5 billion (USD)               211\n",
       "$10 to $25 million (USD)             191\n",
       "$1 to $2 billion (USD)               180\n",
       "$1 to $5 million (USD)               170\n",
       "$25 to $50 million (USD)             143\n",
       "$5 to $10 billion (USD)              133\n",
       "Less than $1 million (USD)           124\n",
       "$500 million to $1 billion (USD)     114\n",
       "$5 to $10 million (USD)              100\n",
       "Name: Revenue, dtype: int64"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Revenue'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "cbd9ae75-40db-4e9d-be69-a2049c536bad",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Revenue'].replace({'-1': 'Others', 'Unknown / Non-Applicable': 'Others'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "30b36fc8-6b55-4795-ae10-d75e0db6e28e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Others                              1392\n",
       "$10+ billion (USD)                   599\n",
       "$100 to $500 million (USD)           338\n",
       "$50 to $100 million (USD)            214\n",
       "$2 to $5 billion (USD)               211\n",
       "$10 to $25 million (USD)             191\n",
       "$1 to $2 billion (USD)               180\n",
       "$1 to $5 million (USD)               170\n",
       "$25 to $50 million (USD)             143\n",
       "$5 to $10 billion (USD)              133\n",
       "Less than $1 million (USD)           124\n",
       "$500 million to $1 billion (USD)     114\n",
       "$5 to $10 million (USD)              100\n",
       "Name: Revenue, dtype: int64"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Revenue'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "ef4ecd7c-9bed-483b-96f8-01685fb9226f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('dataset/data_salary_cleaned.csv') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eef96294-8b84-4350-897f-59ffd03ec262",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}