Movie_review_rotten_tomatoes.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing necessary Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import itertools #to create efficent looping to fetch more data in a go\n",
    "import re \n",
    "import random \n",
    "from textblob import TextBlob"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Movie Urls\n",
    "\n",
    "- https://www.rottentomatoes.com/browse/movies_at_home/audience:upright~critics:fresh?page=5\n",
    "\n",
    "- https://www.rottentomatoes.com/browse/movies_at_home/audience:spilled~critics:fresh?page=5\n",
    "\n",
    "- https://www.rottentomatoes.com/browse/movies_at_home/audience:spilled,upright~critics:fresh?page=5\n",
    "\n",
    "- https://www.rottentomatoes.com/browse/movies_at_home/audience:upright~critics:certified_fresh?page=5\n",
    "\n",
    "- https://www.rottentomatoes.com/browse/movies_at_home/audience:spilled~critics:certified_fresh?page=5\n",
    "\n",
    "- https://www.rottentomatoes.com/browse/movies_at_home/audience:spilled,upright~critics:certified_fresh?page=5\n",
    "\n",
    "- https://www.rottentomatoes.com/browse/movies_at_home/audience:upright~critics:rotten?page=5\n",
    "\n",
    "- https://www.rottentomatoes.com/browse/movies_at_home/audience:spilled~critics:rotten?page=5\n",
    "\n",
    "- https://www.rottentomatoes.com/browse/movies_at_home/audience:spilled,upright~critics:rotten?page=5\n",
    "\n",
    "Here we use page=5 as rottentomatoes will only allow us to check 140 movies at a time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"https://www.rottentomatoes.com/browse/movies_at_home/audience:upright~critics:fresh?page=5\"\n",
    "\n",
    "def getSoup(url):\n",
    "    \"\"\"\n",
    "    Utility function this get soup function will fetch the above url which stored in url var.\n",
    "    \"\"\"\n",
    "    headers = {\n",
    "        'User-Agent': 'Your User-Agent String',\n",
    "        'Authorization': 'Bearer Your_Authentication_Token'  # Include this if authentication is required\n",
    "    }\n",
    "    response = requests.get(url, headers=headers)\n",
    "    soup = BeautifulSoup(response.text, 'html.parser')\n",
    "    return soup\n",
    "\n",
    "def getReviewText(review_url):\n",
    "    '''Returns the user review text given the review url.'''\n",
    "    # find div tags with class text show-more__control\n",
    "    tag = review_url.find('p', attrs={'class': 'review-text'})\n",
    "    return tag.getText()\n",
    "\n",
    "def getMovieTitle(review_url):\n",
    "    '''Returns the movie title from the review url.'''\n",
    "    # find title tag\n",
    "    tag = review_url.find('title')\n",
    "    title_tag = list(tag.children)[0].getText()\n",
    "    \n",
    "    # split the title and remove the unnecessary part\n",
    "    movie_title = title_tag.split(' - Movie Reviews | Rotten Tomatoes')[0]\n",
    "    return movie_title\n",
    "\n",
    "\n",
    "def getNounChunks(user_review):\n",
    "    # create the doc object\n",
    "    doc = nlp(user_review)\n",
    "    # get a list of noun_chunks\n",
    "    noun_chunks = list(doc.noun_chunks)\n",
    "    # convert noun_chunks from span objects to strings, otherwise it won't pick\n",
    "    noun_chunks_strlist = [chunk.text for chunk in noun_chunks]\n",
    "    return noun_chunks_strlist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Filtering the movie tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies_soup = getSoup(url)\n",
    "movie_tags = movies_soup.find_all('a', attrs={'data-qa': \"discovery-media-list-item-caption\"}) + movies_soup.find_all('a', attrs={'class': \"js-tile-link\"})\n",
    "\n",
    "# filter the a-tags to get just the titles\n",
    "movie_links = [tag['href'] for tag in movie_tags]\n",
    "# remove duplicate links\n",
    "unique_movie_links = list(dict.fromkeys(movie_links))\n",
    "\n",
    "print(\"There are a total of \" + str(len(unique_movie_links)) + \" movie titles\")\n",
    "print(\"Displaying 10 titles\")\n",
    "unique_movie_links[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filtering the movie URL's"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "base_url = \"https://www.rottentomatoes.com\"\n",
    "movie_links = [base_url + tag['href'] + '/reviews' for tag in movie_tags]\n",
    "print(\"There are a total of \" + str(len(movie_links)) + \" movie user reviews\")\n",
    "print(\"Displaying 20 user reviews links\")\n",
    "movie_links[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_soups = [getSoup(link) for link in movie_links]\n",
    "# get all movie review links from the 140 listing\n",
    "movie_review_list = [getReviewText(movie_soup) for movie_soup in movie_soups]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Checking how many movie review were able to filter.\n",
    "movie_review_list = list(itertools.chain(*movie_review_list))\n",
    "\n",
    "print(\"There are a total of \" + str(len(movie_review_list)) + \" individual movie reviews\")\n",
    "print(\"Displaying 10 reviews\")\n",
    "print(movie_review_list[:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Converting into the Pandas Data Frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "review_texts = [getReviewText(url) for url in movie_soups]\n",
    "\n",
    "# get movie name from the review link\n",
    "movie_titles = [getMovieTitle(url) for url in movie_soups]\n",
    "print(movie_titles)\n",
    "\n",
    "# Filtering the dataframe with only User_reviews by avoiding links and title\n",
    "\n",
    "# construct a dataframe\n",
    "df = pd.DataFrame({'user_review': review_texts })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head(5) #displaying the resultant data frame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The data frame need to remove index and filter the limit review length by 50 words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_list = [m for m in df['user_review']]\n",
    "#text_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#calculating the length of the text\n",
    "text_list_length = [len(m.split()) for m in text_list]     \n",
    "df['length'] = text_list_length\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[df['length'] < 50]  #limiting the df by 50 in length\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop('length', axis=1, inplace=True)\n",
    "df\n",
    "#dropping the len row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#converting only reviews to CSV & removing the index\n",
    "df.to_csv('data_scrapped/data_rotten_tomatoes.csv', index=False) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Splitting the csv file to the indivitual text files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "\n",
    "with open(\"data_scrapped/data_rotten_tomatoes.csv\", \"r\",encoding=\"utf-8\") as f:\n",
    "        reader = csv.reader(f)\n",
    "        rownumber = 2639    # used to start the naming of the file , change it accordingly \n",
    "        for row in reader:\n",
    "             g=open(str(rownumber)+\".txt\",\"w\")\n",
    "             g.write(str(row))\n",
    "             rownumber = rownumber + 1\n",
    "             g.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def analyze_sentiment(text):\n",
    "    \"\"\"\n",
    "    Analyzes the sentiment of the input text.\n",
    "    \n",
    "    Returns:\n",
    "    - 'positive' if sentiment polarity > 0\n",
    "    - 'negative' if sentiment polarity < 0\n",
    "    - 'neutral' if sentiment polarity == 0\n",
    "    \"\"\"\n",
    "    blob = TextBlob(text)\n",
    "    polarity = blob.sentiment.polarity\n",
    "    \n",
    "    if polarity > 0:\n",
    "        return 'positive'\n",
    "    elif polarity < 0:\n",
    "        return 'negative'\n",
    "    else:\n",
    "        return 'neutral'\n",
    "\n",
    "# Assuming df is your DataFrame containing the reviews\n",
    "df['sentiment'] = df['user_review'].apply(analyze_sentiment)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}