Data_Cleaning_Script.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /Users/ellie/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to /Users/ellie/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import itertools\n",
    "import collections\n",
    "import re\n",
    "import networkx as nx\n",
    "import nltk\n",
    "nltk.download(['punkt','stopwords'])\n",
    "from nltk.corpus import stopwords\n",
    "stopwords = stopwords.words('english')\n",
    "# from nltk.corpus import stopwords\n",
    "# from nltk import bigrams\n",
    "\n",
    "# import warnings\n",
    "# warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "# sns.set(font_scale=1.5)\n",
    "# sns.set_style(\"whitegrid\")\n",
    "# from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer\n",
    "# analyzer = SentimentIntensityAnalyzer()%matplotlib inline\n",
    "# %config InlineBackend.figure_format = 'retina'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>tweet</th>\n",
       "      <th>class</th>\n",
       "      <th>clean_tweet</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>@kooawr watching amas :))</td>\n",
       "      <td>0</td>\n",
       "      <td>@kooawr watching amas :))</td>\n",
       "      <td>@kooawr watching amas :))</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>still can't wrap my head around why this zoo c...</td>\n",
       "      <td>0</td>\n",
       "      <td>still can't wrap head around zoo close i live ...</td>\n",
       "      <td>still can't wrap head around zoo close I live ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>SAVE YOUR TEARS FOR ANOTHER DAY</td>\n",
       "      <td>0</td>\n",
       "      <td>save your tears for another day</td>\n",
       "      <td>SAVE YOUR TEARS FOR ANOTHER DAY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>Good day!!!  #surfing #somewhere #goodwave #en...</td>\n",
       "      <td>0</td>\n",
       "      <td>good day!!! #surfing #somewhere #goodwave #enj...</td>\n",
       "      <td>Good day!!! #surfing #somewhere #goodwave #enj...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>@poppingin50 have a nice day yh</td>\n",
       "      <td>0</td>\n",
       "      <td>@poppingin50 nice day yh</td>\n",
       "      <td>@poppingin50 nice day yh</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1395</th>\n",
       "      <td>1395</td>\n",
       "      <td>95</td>\n",
       "      <td>But in the meantime...? Our basic needs are be...</td>\n",
       "      <td>1</td>\n",
       "      <td>but meantime...? our basic needs met. we're st...</td>\n",
       "      <td>But meantime...? Our basic needs MET. We're st...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1396</th>\n",
       "      <td>1396</td>\n",
       "      <td>96</td>\n",
       "      <td>@voltruto kaya pala trending ka always ning HH...</td>\n",
       "      <td>1</td>\n",
       "      <td>@voltruto kaya pala trending ka always ning hh...</td>\n",
       "      <td>@voltruto kaya pala trending ka always ning HH...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1397</th>\n",
       "      <td>1397</td>\n",
       "      <td>97</td>\n",
       "      <td>the more illegal the better i always say  its hot</td>\n",
       "      <td>1</td>\n",
       "      <td>illegal better always say hot</td>\n",
       "      <td>illegal better always say hot</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1398</th>\n",
       "      <td>1398</td>\n",
       "      <td>98</td>\n",
       "      <td>@simplysarris i feel bad even if i mute someon...</td>\n",
       "      <td>1</td>\n",
       "      <td>@simplysarris feel bad even mute someone 🙃 alw...</td>\n",
       "      <td>@simplysarris feel bad even mute someone 🙃 alw...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1399</th>\n",
       "      <td>1399</td>\n",
       "      <td>99</td>\n",
       "      <td>@dyutka LMFAO SAME OMG i used to go to the one...</td>\n",
       "      <td>1</td>\n",
       "      <td>@dyutka lmfao same omg used go one jamaica itt...</td>\n",
       "      <td>@dyutka LMFAO SAME OMG used go one jamaica itt...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1400 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0  Unnamed: 0.1  \\\n",
       "0              0             0   \n",
       "1              1             1   \n",
       "2              2             2   \n",
       "3              3             3   \n",
       "4              4             4   \n",
       "...          ...           ...   \n",
       "1395        1395            95   \n",
       "1396        1396            96   \n",
       "1397        1397            97   \n",
       "1398        1398            98   \n",
       "1399        1399            99   \n",
       "\n",
       "                                                  tweet  class  \\\n",
       "0                             @kooawr watching amas :))      0   \n",
       "1     still can't wrap my head around why this zoo c...      0   \n",
       "2                       SAVE YOUR TEARS FOR ANOTHER DAY      0   \n",
       "3     Good day!!!  #surfing #somewhere #goodwave #en...      0   \n",
       "4                       @poppingin50 have a nice day yh      0   \n",
       "...                                                 ...    ...   \n",
       "1395  But in the meantime...? Our basic needs are be...      1   \n",
       "1396  @voltruto kaya pala trending ka always ning HH...      1   \n",
       "1397  the more illegal the better i always say  its hot      1   \n",
       "1398  @simplysarris i feel bad even if i mute someon...      1   \n",
       "1399  @dyutka LMFAO SAME OMG i used to go to the one...      1   \n",
       "\n",
       "                                            clean_tweet  \\\n",
       "0                             @kooawr watching amas :))   \n",
       "1     still can't wrap head around zoo close i live ...   \n",
       "2                       save your tears for another day   \n",
       "3     good day!!! #surfing #somewhere #goodwave #enj...   \n",
       "4                              @poppingin50 nice day yh   \n",
       "...                                                 ...   \n",
       "1395  but meantime...? our basic needs met. we're st...   \n",
       "1396  @voltruto kaya pala trending ka always ning hh...   \n",
       "1397                      illegal better always say hot   \n",
       "1398  @simplysarris feel bad even mute someone 🙃 alw...   \n",
       "1399  @dyutka lmfao same omg used go one jamaica itt...   \n",
       "\n",
       "                                                   text  \n",
       "0                             @kooawr watching amas :))  \n",
       "1     still can't wrap head around zoo close I live ...  \n",
       "2                       SAVE YOUR TEARS FOR ANOTHER DAY  \n",
       "3     Good day!!! #surfing #somewhere #goodwave #enj...  \n",
       "4                              @poppingin50 nice day yh  \n",
       "...                                                 ...  \n",
       "1395  But meantime...? Our basic needs MET. We're st...  \n",
       "1396  @voltruto kaya pala trending ka always ning HH...  \n",
       "1397                      illegal better always say hot  \n",
       "1398  @simplysarris feel bad even mute someone 🙃 alw...  \n",
       "1399  @dyutka LMFAO SAME OMG used go one jamaica itt...  \n",
       "\n",
       "[1400 rows x 6 columns]"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_df = pd.read_csv('dataset_to_train.csv')\n",
    "\n",
    "dataset_df = dataset_df.drop_duplicates()\n",
    "dataset_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_df['clean_tweet'] = dataset_df['tweet'].apply(lambda x: ' '.join([item for item in x.split() if item not in stopwords]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>tweet</th>\n",
       "      <th>class</th>\n",
       "      <th>clean_tweet</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>@kooawr watching amas :))</td>\n",
       "      <td>0</td>\n",
       "      <td>@kooawr watching amas :))</td>\n",
       "      <td>@kooawr watching amas :))</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>still can't wrap my head around why this zoo c...</td>\n",
       "      <td>0</td>\n",
       "      <td>still can't wrap head around zoo close I live ...</td>\n",
       "      <td>still can't wrap head around zoo close I live ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>SAVE YOUR TEARS FOR ANOTHER DAY</td>\n",
       "      <td>0</td>\n",
       "      <td>SAVE YOUR TEARS FOR ANOTHER DAY</td>\n",
       "      <td>SAVE YOUR TEARS FOR ANOTHER DAY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>Good day!!!  #surfing #somewhere #goodwave #en...</td>\n",
       "      <td>0</td>\n",
       "      <td>Good day!!! #surfing #somewhere #goodwave #enj...</td>\n",
       "      <td>Good day!!! #surfing #somewhere #goodwave #enj...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>@poppingin50 have a nice day yh</td>\n",
       "      <td>0</td>\n",
       "      <td>@poppingin50 nice day yh</td>\n",
       "      <td>@poppingin50 nice day yh</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1395</th>\n",
       "      <td>1395</td>\n",
       "      <td>95</td>\n",
       "      <td>But in the meantime...? Our basic needs are be...</td>\n",
       "      <td>1</td>\n",
       "      <td>But meantime...? Our basic needs MET. We're st...</td>\n",
       "      <td>But meantime...? Our basic needs MET. We're st...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1396</th>\n",
       "      <td>1396</td>\n",
       "      <td>96</td>\n",
       "      <td>@voltruto kaya pala trending ka always ning HH...</td>\n",
       "      <td>1</td>\n",
       "      <td>@voltruto kaya pala trending ka always ning HH...</td>\n",
       "      <td>@voltruto kaya pala trending ka always ning HH...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1397</th>\n",
       "      <td>1397</td>\n",
       "      <td>97</td>\n",
       "      <td>the more illegal the better i always say  its hot</td>\n",
       "      <td>1</td>\n",
       "      <td>illegal better always say hot</td>\n",
       "      <td>illegal better always say hot</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1398</th>\n",
       "      <td>1398</td>\n",
       "      <td>98</td>\n",
       "      <td>@simplysarris i feel bad even if i mute someon...</td>\n",
       "      <td>1</td>\n",
       "      <td>@simplysarris feel bad even mute someone 🙃 alw...</td>\n",
       "      <td>@simplysarris feel bad even mute someone 🙃 alw...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1399</th>\n",
       "      <td>1399</td>\n",
       "      <td>99</td>\n",
       "      <td>@dyutka LMFAO SAME OMG i used to go to the one...</td>\n",
       "      <td>1</td>\n",
       "      <td>@dyutka LMFAO SAME OMG used go one jamaica itt...</td>\n",
       "      <td>@dyutka LMFAO SAME OMG used go one jamaica itt...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1400 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0  Unnamed: 0.1  \\\n",
       "0              0             0   \n",
       "1              1             1   \n",
       "2              2             2   \n",
       "3              3             3   \n",
       "4              4             4   \n",
       "...          ...           ...   \n",
       "1395        1395            95   \n",
       "1396        1396            96   \n",
       "1397        1397            97   \n",
       "1398        1398            98   \n",
       "1399        1399            99   \n",
       "\n",
       "                                                  tweet  class  \\\n",
       "0                             @kooawr watching amas :))      0   \n",
       "1     still can't wrap my head around why this zoo c...      0   \n",
       "2                       SAVE YOUR TEARS FOR ANOTHER DAY      0   \n",
       "3     Good day!!!  #surfing #somewhere #goodwave #en...      0   \n",
       "4                       @poppingin50 have a nice day yh      0   \n",
       "...                                                 ...    ...   \n",
       "1395  But in the meantime...? Our basic needs are be...      1   \n",
       "1396  @voltruto kaya pala trending ka always ning HH...      1   \n",
       "1397  the more illegal the better i always say  its hot      1   \n",
       "1398  @simplysarris i feel bad even if i mute someon...      1   \n",
       "1399  @dyutka LMFAO SAME OMG i used to go to the one...      1   \n",
       "\n",
       "                                            clean_tweet  \\\n",
       "0                             @kooawr watching amas :))   \n",
       "1     still can't wrap head around zoo close I live ...   \n",
       "2                       SAVE YOUR TEARS FOR ANOTHER DAY   \n",
       "3     Good day!!! #surfing #somewhere #goodwave #enj...   \n",
       "4                              @poppingin50 nice day yh   \n",
       "...                                                 ...   \n",
       "1395  But meantime...? Our basic needs MET. We're st...   \n",
       "1396  @voltruto kaya pala trending ka always ning HH...   \n",
       "1397                      illegal better always say hot   \n",
       "1398  @simplysarris feel bad even mute someone 🙃 alw...   \n",
       "1399  @dyutka LMFAO SAME OMG used go one jamaica itt...   \n",
       "\n",
       "                                                   text  \n",
       "0                             @kooawr watching amas :))  \n",
       "1     still can't wrap head around zoo close I live ...  \n",
       "2                       SAVE YOUR TEARS FOR ANOTHER DAY  \n",
       "3     Good day!!! #surfing #somewhere #goodwave #enj...  \n",
       "4                              @poppingin50 nice day yh  \n",
       "...                                                 ...  \n",
       "1395  But meantime...? Our basic needs MET. We're st...  \n",
       "1396  @voltruto kaya pala trending ka always ning HH...  \n",
       "1397                      illegal better always say hot  \n",
       "1398  @simplysarris feel bad even mute someone 🙃 alw...  \n",
       "1399  @dyutka LMFAO SAME OMG used go one jamaica itt...  \n",
       "\n",
       "[1400 rows x 6 columns]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                               @kooawr watching amas :))\n",
       "1       still can't wrap head around zoo close I live ...\n",
       "2                         SAVE YOUR TEARS FOR ANOTHER DAY\n",
       "3       Good day!!! #surfing #somewhere #goodwave #enj...\n",
       "4                                @poppingin50 nice day yh\n",
       "                              ...                        \n",
       "1395    But meantime...? Our basic needs MET. We're st...\n",
       "1396    @voltruto kaya pala trending ka always ning HH...\n",
       "1397                        illegal better always say hot\n",
       "1398    @simplysarris feel bad even mute someone 🙃 alw...\n",
       "1399    @dyutka LMFAO SAME OMG used go one jamaica itt...\n",
       "Name: text, Length: 1400, dtype: object"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_df['text'] = dataset_df['clean_tweet']\n",
    "dataset_df['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['kooawr watching amas',\n",
       " 'still cant wrap head around zoo close I live phantom cave dude playing him hed appear certain times day went get stamp hand ring',\n",
       " 'SAVE YOUR TEARS FOR ANOTHER DAY',\n",
       " 'Good day surfing somewhere goodwave enjoy Timmys Cafe',\n",
       " 'poppingin50 nice day yh']"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def remove_url(txt):\n",
    "    return \" \".join(re.sub(\"([^0-9A-Za-z \\t])|(\\w+:\\/\\/\\S+)\", \"\", txt).split())\n",
    "all_tweets_no_urls = [remove_url(tweet) for tweet in dataset_df['text']]\n",
    "all_tweets_no_urls[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "#lower_case = [word.lower() for word in df_new['text']]\n",
    "dataset_df[\"clean_tweet\"]=dataset_df[\"clean_tweet\"].str.lower()\n",
    "dataset_df['clean_tweet'] = dataset_df['clean_tweet'].apply(lambda x: re.split('https:\\/\\/.*', str(x))[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1400 entries, 0 to 1399\n",
      "Data columns (total 6 columns):\n",
      " #   Column        Non-Null Count  Dtype \n",
      "---  ------        --------------  ----- \n",
      " 0   Unnamed: 0    1400 non-null   int64 \n",
      " 1   Unnamed: 0.1  1400 non-null   int64 \n",
      " 2   tweet         1400 non-null   object\n",
      " 3   class         1400 non-null   int64 \n",
      " 4   clean_tweet   1400 non-null   object\n",
      " 5   text          1400 non-null   object\n",
      "dtypes: int64(3), object(3)\n",
      "memory usage: 76.6+ KB\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1400 entries, 0 to 1399\n",
      "Data columns (total 6 columns):\n",
      " #   Column        Non-Null Count  Dtype \n",
      "---  ------        --------------  ----- \n",
      " 0   Unnamed: 0    1400 non-null   int64 \n",
      " 1   Unnamed: 0.1  1400 non-null   int64 \n",
      " 2   tweet         1400 non-null   object\n",
      " 3   class         1400 non-null   int64 \n",
      " 4   clean_tweet   1400 non-null   object\n",
      " 5   text          1400 non-null   object\n",
      "dtypes: int64(3), object(3)\n",
      "memory usage: 76.6+ KB\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(dataset_df.isnull().any().any())\n",
    "## No null values in our dataset- can ignore removing\n",
    "dataset_df.info(null_counts=True)\n",
    "df_new = dataset_df[dataset_df['clean_tweet'].notnull()]\n",
    "df_new.info(null_counts=True)\n",
    "df_new.isnull().any().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "users = df_new['clean_tweet'].str.startswith('@')\n",
    "#dataset_df['clean_tweet'] = dataset_df['clean_tweet'].str.strip('@') \n",
    "#dataset_df\n",
    "users\n",
    "df_new=df_new[~users].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_new\n",
    "\n",
    "df_new.to_csv(r'/Users/ellie/Desktop/dataset_to_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>tweet</th>\n",
       "      <th>class</th>\n",
       "      <th>clean_tweet</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>still can't wrap my head around why this zoo c...</td>\n",
       "      <td>0</td>\n",
       "      <td>still can't wrap head around zoo close i live ...</td>\n",
       "      <td>still can't wrap head around zoo close I live ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>SAVE YOUR TEARS FOR ANOTHER DAY</td>\n",
       "      <td>0</td>\n",
       "      <td>save your tears for another day</td>\n",
       "      <td>SAVE YOUR TEARS FOR ANOTHER DAY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>Good day!!!  #surfing #somewhere #goodwave #en...</td>\n",
       "      <td>0</td>\n",
       "      <td>good day!!! #surfing #somewhere #goodwave #enj...</td>\n",
       "      <td>Good day!!! #surfing #somewhere #goodwave #enj...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>Uri Hyeop singing Spring Day - BTS Gosh! I nee...</td>\n",
       "      <td>0</td>\n",
       "      <td>uri hyeop singing spring day - bts gosh! i nee...</td>\n",
       "      <td>Uri Hyeop singing Spring Day - BTS Gosh! I nee...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>Today is just be mean to kleo day I see.</td>\n",
       "      <td>0</td>\n",
       "      <td>today mean kleo day i see.</td>\n",
       "      <td>Today mean kleo day I see.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>781</th>\n",
       "      <td>1386</td>\n",
       "      <td>86</td>\n",
       "      <td>Thank you @Hersheys for always having this com...</td>\n",
       "      <td>1</td>\n",
       "      <td>thank @hersheys always commercial! it’s favori...</td>\n",
       "      <td>Thank @Hersheys always commercial! It’s FAVORI...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>782</th>\n",
       "      <td>1389</td>\n",
       "      <td>89</td>\n",
       "      <td>My heart. ♥ I will love him until the end of.....</td>\n",
       "      <td>1</td>\n",
       "      <td>my heart. ♥ i love end of... i always love him.</td>\n",
       "      <td>My heart. ♥ I love end of... I always love him.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>783</th>\n",
       "      <td>1393</td>\n",
       "      <td>93</td>\n",
       "      <td>Deathstroke would have voted for him in 2016 a...</td>\n",
       "      <td>1</td>\n",
       "      <td>deathstroke would voted 2016 2020. he'd jump s...</td>\n",
       "      <td>Deathstroke would voted 2016 2020. He'd jump s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>784</th>\n",
       "      <td>1395</td>\n",
       "      <td>95</td>\n",
       "      <td>But in the meantime...? Our basic needs are be...</td>\n",
       "      <td>1</td>\n",
       "      <td>but meantime...? our basic needs met. we're st...</td>\n",
       "      <td>But meantime...? Our basic needs MET. We're st...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>785</th>\n",
       "      <td>1397</td>\n",
       "      <td>97</td>\n",
       "      <td>the more illegal the better i always say  its hot</td>\n",
       "      <td>1</td>\n",
       "      <td>illegal better always say hot</td>\n",
       "      <td>illegal better always say hot</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>786 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Unnamed: 0  Unnamed: 0.1  \\\n",
       "0             1             1   \n",
       "1             2             2   \n",
       "2             3             3   \n",
       "3             6             6   \n",
       "4             7             7   \n",
       "..          ...           ...   \n",
       "781        1386            86   \n",
       "782        1389            89   \n",
       "783        1393            93   \n",
       "784        1395            95   \n",
       "785        1397            97   \n",
       "\n",
       "                                                 tweet  class  \\\n",
       "0    still can't wrap my head around why this zoo c...      0   \n",
       "1                      SAVE YOUR TEARS FOR ANOTHER DAY      0   \n",
       "2    Good day!!!  #surfing #somewhere #goodwave #en...      0   \n",
       "3    Uri Hyeop singing Spring Day - BTS Gosh! I nee...      0   \n",
       "4             Today is just be mean to kleo day I see.      0   \n",
       "..                                                 ...    ...   \n",
       "781  Thank you @Hersheys for always having this com...      1   \n",
       "782  My heart. ♥ I will love him until the end of.....      1   \n",
       "783  Deathstroke would have voted for him in 2016 a...      1   \n",
       "784  But in the meantime...? Our basic needs are be...      1   \n",
       "785  the more illegal the better i always say  its hot      1   \n",
       "\n",
       "                                           clean_tweet  \\\n",
       "0    still can't wrap head around zoo close i live ...   \n",
       "1                      save your tears for another day   \n",
       "2    good day!!! #surfing #somewhere #goodwave #enj...   \n",
       "3    uri hyeop singing spring day - bts gosh! i nee...   \n",
       "4                           today mean kleo day i see.   \n",
       "..                                                 ...   \n",
       "781  thank @hersheys always commercial! it’s favori...   \n",
       "782    my heart. ♥ i love end of... i always love him.   \n",
       "783  deathstroke would voted 2016 2020. he'd jump s...   \n",
       "784  but meantime...? our basic needs met. we're st...   \n",
       "785                      illegal better always say hot   \n",
       "\n",
       "                                                  text  \n",
       "0    still can't wrap head around zoo close I live ...  \n",
       "1                      SAVE YOUR TEARS FOR ANOTHER DAY  \n",
       "2    Good day!!! #surfing #somewhere #goodwave #enj...  \n",
       "3    Uri Hyeop singing Spring Day - BTS Gosh! I nee...  \n",
       "4                           Today mean kleo day I see.  \n",
       "..                                                 ...  \n",
       "781  Thank @Hersheys always commercial! It’s FAVORI...  \n",
       "782    My heart. ♥ I love end of... I always love him.  \n",
       "783  Deathstroke would voted 2016 2020. He'd jump s...  \n",
       "784  But meantime...? Our basic needs MET. We're st...  \n",
       "785                      illegal better always say hot  \n",
       "\n",
       "[786 rows x 6 columns]"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}