Data Cleaning and Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ead0cca3-7713-4c44-adbe-9bab5c25eedb",
   "metadata": {},
   "source": [
    "# Spark  Instantiation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9ae53d8c-a3c4-4503-af4a-035144eeb37e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'3.1.3'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Ensure we are using the right kernel\n",
    "spark.version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3295a6d6-fe77-4dc1-bb60-b8f4d560e53c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import subprocess\n",
    "import shutil\n",
    "import pandas as pd\n",
    "# import sh\n",
    "from pyspark.sql.functions import *\n",
    "#from pyspark.sql import functions as F\n",
    "from pyspark.sql.types import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f7682d0d-4361-4b57-bdf4-25b1f0b8b4f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import subprocess\n",
    "\n",
    "from pyspark.sql.functions import *\n",
    "from pyspark.sql.types import *\n",
    "\n",
    "from pyspark.sql import SQLContext\n",
    "sqlContext = SQLContext(sc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "19444bc5-06ae-49ba-acdb-9b47e1f066ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option(\"max_colwidth\", 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "bfe6db97-33c7-4a21-8902-f9ae867d7da1",
   "metadata": {},
   "outputs": [],
   "source": [
    "warnings.filterwarnings(action='ignore')\n",
    "spark = SparkSession.builder.getOrCreate()\n",
    "\n",
    "##Add \"eagerEval.enabled\" to beautify the way Spark DF is displayed\n",
    "spark.conf.set(\"spark.sql.repl.eagerEval.enabled\",True)\n",
    "\n",
    "## To use legacy casting notation for date\n",
    "spark.conf.set(\"spark.sql.legacy.timeParserPolicy\",\"LEGACY\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ef495c9-e1a9-426b-95a8-e3a778550d95",
   "metadata": {},
   "source": [
    "# Loading the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2aee859c-6223-44ec-98eb-02c61233a7f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.cloud import storage\n",
    "# !pip install gcsfs --upgrade"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6828cf58-6ac2-4e79-96b7-c2ae31196094",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"gs://msca-bdp-tweets/final_project/\"\n",
    "bucket = 'msca-bdp-tweets/final_project'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c7e9dd86-0fc1-4427-87ec-1fa1f60e81f4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 50696 items\n",
      "-rwx------   3 root root          0 2023-02-08 13:58 gs://msca-bdp-tweets/final_project/_SUCCESS\n",
      "-rwx------   3 root root    4500466 2023-02-08 13:44 gs://msca-bdp-tweets/final_project/part-00000-aa6d3cb4-7022-4df2-9921-218307589ce2-c000.json\n",
      "-rwx------   3 root root    4107431 2023-02-08 13:44 gs://msca-bdp-tweets/final_project/part-00001-aa6d3cb4-7022-4df2-9921-218307589ce2-c000.json\n",
      "-rwx------   3 root root    4672123 2023-02-08 13:44 gs://msca-bdp-tweets/final_project/part-00002-aa6d3cb4-7022-4df2-9921-218307589ce2-c000.json\n",
      "-rwx------   3 root root    5186684 2023-02-08 13:44 gs://msca-bdp-tweets/final_project/part-00003-aa6d3cb4-7022-4df2-9921-218307589ce2-c000.json\n",
      "-rwx------   3 root root    4729662 2023-02-08 13:44 gs://msca-bdp-tweets/final_project/part-00004-aa6d3cb4-7022-4df2-9921-218307589ce2-c000.json\n",
      "-rwx------   3 root root    4605529 2023-02-08 13:44 gs://msca-bdp-tweets/final_project/part-00005-aa6d3cb4-7022-4df2-9921-218307589ce2-c000.json\n",
      "-rwx------   3 root root    6370756 2023-02-08 13:44 gs://msca-bdp-tweets/final_project/part-00006-aa6d3cb4-7022-4df2-9921-218307589ce2-c000.json\n",
      "-rwx------   3 root root    4894125 2023-02-08 13:44 gs://msca-bdp-tweets/final_project/part-00007-aa6d3cb4-7022-4df2-9921-218307589ce2-c000.json\n"
     ]
    }
   ],
   "source": [
    "!hadoop fs -ls 'gs://msca-bdp-tweets/final_project/' | head -10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9e05aed3-7bdc-42c9-b5f8-57317dd50635",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3 µs, sys: 1 µs, total: 4 µs\n",
      "Wall time: 7.63 µs\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23/02/27 14:30:48 WARN org.apache.spark.sql.execution.datasources.SharedInMemoryCache: Evicting cached table partition metadata from memory due to size constraints (spark.sql.hive.filesourcePartitionFileCacheSize = 262144000 bytes). This may impact query planning performance.\n",
      "23/02/27 14:36:47 WARN org.apache.spark.sql.catalyst.util.package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n"
     ]
    }
   ],
   "source": [
    "%time\n",
    "tweets_df = spark.read.json(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ed52a31d-8247-4cfc-b6a9-f018fa9a997b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- coordinates: struct (nullable = true)\n",
      " |    |-- coordinates: array (nullable = true)\n",
      " |    |    |-- element: double (containsNull = true)\n",
      " |    |-- type: string (nullable = true)\n",
      " |-- created_at: string (nullable = true)\n",
      " |-- display_text_range: array (nullable = true)\n",
      " |    |-- element: long (containsNull = true)\n",
      " |-- entities: struct (nullable = true)\n",
      " |    |-- hashtags: array (nullable = true)\n",
      " |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |-- text: string (nullable = true)\n",
      " |    |-- media: array (nullable = true)\n",
      " |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |-- url: string (nullable = true)\n",
      " |    |-- symbols: array (nullable = true)\n",
      " |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |-- text: string (nullable = true)\n",
      " |    |-- urls: array (nullable = true)\n",
      " |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |-- url: string (nullable = true)\n",
      " |    |-- user_mentions: array (nullable = true)\n",
      " |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |-- name: string (nullable = true)\n",
      " |    |    |    |-- screen_name: string (nullable = true)\n",
      " |-- extended_entities: struct (nullable = true)\n",
      " |    |-- media: array (nullable = true)\n",
      " |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |-- video_info: struct (nullable = true)\n",
      " |    |    |    |    |-- aspect_ratio: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- duration_millis: long (nullable = true)\n",
      " |    |    |    |    |-- variants: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |-- bitrate: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- content_type: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |-- extended_tweet: struct (nullable = true)\n",
      " |    |-- display_text_range: array (nullable = true)\n",
      " |    |    |-- element: long (containsNull = true)\n",
      " |    |-- entities: struct (nullable = true)\n",
      " |    |    |-- hashtags: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- text: string (nullable = true)\n",
      " |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |    |-- video_info: struct (nullable = true)\n",
      " |    |    |    |    |    |-- aspect_ratio: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- duration_millis: long (nullable = true)\n",
      " |    |    |    |    |    |-- variants: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |    |-- bitrate: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- content_type: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |-- symbols: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- text: string (nullable = true)\n",
      " |    |    |-- urls: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |-- user_mentions: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- name: string (nullable = true)\n",
      " |    |    |    |    |-- screen_name: string (nullable = true)\n",
      " |    |-- extended_entities: struct (nullable = true)\n",
      " |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |    |-- video_info: struct (nullable = true)\n",
      " |    |    |    |    |    |-- aspect_ratio: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- duration_millis: long (nullable = true)\n",
      " |    |    |    |    |    |-- variants: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |    |-- bitrate: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- content_type: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |-- full_text: string (nullable = true)\n",
      " |-- favorite_count: long (nullable = true)\n",
      " |-- favorited: boolean (nullable = true)\n",
      " |-- filter_level: string (nullable = true)\n",
      " |-- geo: struct (nullable = true)\n",
      " |    |-- coordinates: array (nullable = true)\n",
      " |    |    |-- element: double (containsNull = true)\n",
      " |    |-- type: string (nullable = true)\n",
      " |-- id: long (nullable = true)\n",
      " |-- id_str: string (nullable = true)\n",
      " |-- in_reply_to_screen_name: string (nullable = true)\n",
      " |-- in_reply_to_status_id: long (nullable = true)\n",
      " |-- in_reply_to_status_id_str: string (nullable = true)\n",
      " |-- in_reply_to_user_id: long (nullable = true)\n",
      " |-- in_reply_to_user_id_str: string (nullable = true)\n",
      " |-- is_quote_status: boolean (nullable = true)\n",
      " |-- lang: string (nullable = true)\n",
      " |-- place: struct (nullable = true)\n",
      " |    |-- bounding_box: struct (nullable = true)\n",
      " |    |    |-- coordinates: array (nullable = true)\n",
      " |    |    |    |-- element: array (containsNull = true)\n",
      " |    |    |    |    |-- element: array (containsNull = true)\n",
      " |    |    |    |    |    |-- element: double (containsNull = true)\n",
      " |    |    |-- type: string (nullable = true)\n",
      " |    |-- country: string (nullable = true)\n",
      " |    |-- country_code: string (nullable = true)\n",
      " |    |-- full_name: string (nullable = true)\n",
      " |    |-- id: string (nullable = true)\n",
      " |    |-- name: string (nullable = true)\n",
      " |    |-- place_type: string (nullable = true)\n",
      " |    |-- url: string (nullable = true)\n",
      " |-- possibly_sensitive: boolean (nullable = true)\n",
      " |-- quote_count: long (nullable = true)\n",
      " |-- quoted_status: struct (nullable = true)\n",
      " |    |-- coordinates: struct (nullable = true)\n",
      " |    |    |-- coordinates: array (nullable = true)\n",
      " |    |    |    |-- element: double (containsNull = true)\n",
      " |    |    |-- type: string (nullable = true)\n",
      " |    |-- created_at: string (nullable = true)\n",
      " |    |-- display_text_range: array (nullable = true)\n",
      " |    |    |-- element: long (containsNull = true)\n",
      " |    |-- entities: struct (nullable = true)\n",
      " |    |    |-- hashtags: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- text: string (nullable = true)\n",
      " |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |-- symbols: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- text: string (nullable = true)\n",
      " |    |    |-- urls: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |-- user_mentions: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- name: string (nullable = true)\n",
      " |    |    |    |    |-- screen_name: string (nullable = true)\n",
      " |    |-- extended_entities: struct (nullable = true)\n",
      " |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |    |-- video_info: struct (nullable = true)\n",
      " |    |    |    |    |    |-- aspect_ratio: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- duration_millis: long (nullable = true)\n",
      " |    |    |    |    |    |-- variants: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |    |-- bitrate: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- content_type: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |-- extended_tweet: struct (nullable = true)\n",
      " |    |    |-- display_text_range: array (nullable = true)\n",
      " |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |-- entities: struct (nullable = true)\n",
      " |    |    |    |-- hashtags: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- text: string (nullable = true)\n",
      " |    |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |    |    |-- video_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- aspect_ratio: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |    |-- duration_millis: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- variants: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |    |    |-- bitrate: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- content_type: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |-- symbols: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- text: string (nullable = true)\n",
      " |    |    |    |-- urls: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |-- user_mentions: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- name: string (nullable = true)\n",
      " |    |    |    |    |    |-- screen_name: string (nullable = true)\n",
      " |    |    |-- extended_entities: struct (nullable = true)\n",
      " |    |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |    |    |-- video_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- aspect_ratio: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |    |-- duration_millis: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- variants: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |    |    |-- bitrate: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- content_type: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |-- full_text: string (nullable = true)\n",
      " |    |-- favorite_count: long (nullable = true)\n",
      " |    |-- favorited: boolean (nullable = true)\n",
      " |    |-- filter_level: string (nullable = true)\n",
      " |    |-- geo: struct (nullable = true)\n",
      " |    |    |-- coordinates: array (nullable = true)\n",
      " |    |    |    |-- element: double (containsNull = true)\n",
      " |    |    |-- type: string (nullable = true)\n",
      " |    |-- id: long (nullable = true)\n",
      " |    |-- id_str: string (nullable = true)\n",
      " |    |-- in_reply_to_screen_name: string (nullable = true)\n",
      " |    |-- in_reply_to_status_id: long (nullable = true)\n",
      " |    |-- in_reply_to_status_id_str: string (nullable = true)\n",
      " |    |-- in_reply_to_user_id: long (nullable = true)\n",
      " |    |-- in_reply_to_user_id_str: string (nullable = true)\n",
      " |    |-- is_quote_status: boolean (nullable = true)\n",
      " |    |-- lang: string (nullable = true)\n",
      " |    |-- place: struct (nullable = true)\n",
      " |    |    |-- bounding_box: struct (nullable = true)\n",
      " |    |    |    |-- coordinates: array (nullable = true)\n",
      " |    |    |    |    |-- element: array (containsNull = true)\n",
      " |    |    |    |    |    |-- element: array (containsNull = true)\n",
      " |    |    |    |    |    |    |-- element: double (containsNull = true)\n",
      " |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |-- country: string (nullable = true)\n",
      " |    |    |-- country_code: string (nullable = true)\n",
      " |    |    |-- full_name: string (nullable = true)\n",
      " |    |    |-- id: string (nullable = true)\n",
      " |    |    |-- name: string (nullable = true)\n",
      " |    |    |-- place_type: string (nullable = true)\n",
      " |    |    |-- url: string (nullable = true)\n",
      " |    |-- possibly_sensitive: boolean (nullable = true)\n",
      " |    |-- quote_count: long (nullable = true)\n",
      " |    |-- quoted_status_id: long (nullable = true)\n",
      " |    |-- quoted_status_id_str: string (nullable = true)\n",
      " |    |-- reply_count: long (nullable = true)\n",
      " |    |-- retweet_count: long (nullable = true)\n",
      " |    |-- retweeted: boolean (nullable = true)\n",
      " |    |-- scopes: struct (nullable = true)\n",
      " |    |    |-- followers: boolean (nullable = true)\n",
      " |    |-- source: string (nullable = true)\n",
      " |    |-- text: string (nullable = true)\n",
      " |    |-- truncated: boolean (nullable = true)\n",
      " |    |-- user: struct (nullable = true)\n",
      " |    |    |-- contributors_enabled: boolean (nullable = true)\n",
      " |    |    |-- created_at: string (nullable = true)\n",
      " |    |    |-- default_profile: boolean (nullable = true)\n",
      " |    |    |-- default_profile_image: boolean (nullable = true)\n",
      " |    |    |-- description: string (nullable = true)\n",
      " |    |    |-- favourites_count: long (nullable = true)\n",
      " |    |    |-- followers_count: long (nullable = true)\n",
      " |    |    |-- friends_count: long (nullable = true)\n",
      " |    |    |-- geo_enabled: boolean (nullable = true)\n",
      " |    |    |-- id: long (nullable = true)\n",
      " |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |-- is_translator: boolean (nullable = true)\n",
      " |    |    |-- listed_count: long (nullable = true)\n",
      " |    |    |-- location: string (nullable = true)\n",
      " |    |    |-- name: string (nullable = true)\n",
      " |    |    |-- profile_background_color: string (nullable = true)\n",
      " |    |    |-- profile_background_image_url: string (nullable = true)\n",
      " |    |    |-- profile_background_image_url_https: string (nullable = true)\n",
      " |    |    |-- profile_background_tile: boolean (nullable = true)\n",
      " |    |    |-- profile_banner_url: string (nullable = true)\n",
      " |    |    |-- profile_image_url: string (nullable = true)\n",
      " |    |    |-- profile_image_url_https: string (nullable = true)\n",
      " |    |    |-- profile_link_color: string (nullable = true)\n",
      " |    |    |-- profile_sidebar_border_color: string (nullable = true)\n",
      " |    |    |-- profile_sidebar_fill_color: string (nullable = true)\n",
      " |    |    |-- profile_text_color: string (nullable = true)\n",
      " |    |    |-- profile_use_background_image: boolean (nullable = true)\n",
      " |    |    |-- protected: boolean (nullable = true)\n",
      " |    |    |-- screen_name: string (nullable = true)\n",
      " |    |    |-- statuses_count: long (nullable = true)\n",
      " |    |    |-- translator_type: string (nullable = true)\n",
      " |    |    |-- url: string (nullable = true)\n",
      " |    |    |-- verified: boolean (nullable = true)\n",
      " |    |    |-- verified_type: string (nullable = true)\n",
      " |    |    |-- withheld_in_countries: array (nullable = true)\n",
      " |    |    |    |-- element: string (containsNull = true)\n",
      " |    |-- withheld_copyright: boolean (nullable = true)\n",
      " |    |-- withheld_in_countries: array (nullable = true)\n",
      " |    |    |-- element: string (containsNull = true)\n",
      " |-- quoted_status_id: long (nullable = true)\n",
      " |-- quoted_status_id_str: string (nullable = true)\n",
      " |-- quoted_status_permalink: struct (nullable = true)\n",
      " |    |-- display: string (nullable = true)\n",
      " |    |-- expanded: string (nullable = true)\n",
      " |    |-- url: string (nullable = true)\n",
      " |-- quoted_text: string (nullable = true)\n",
      " |-- reply_count: long (nullable = true)\n",
      " |-- retweet_count: long (nullable = true)\n",
      " |-- retweeted: string (nullable = true)\n",
      " |-- retweeted_from: string (nullable = true)\n",
      " |-- retweeted_status: struct (nullable = true)\n",
      " |    |-- coordinates: struct (nullable = true)\n",
      " |    |    |-- coordinates: array (nullable = true)\n",
      " |    |    |    |-- element: double (containsNull = true)\n",
      " |    |    |-- type: string (nullable = true)\n",
      " |    |-- created_at: string (nullable = true)\n",
      " |    |-- display_text_range: array (nullable = true)\n",
      " |    |    |-- element: long (containsNull = true)\n",
      " |    |-- entities: struct (nullable = true)\n",
      " |    |    |-- hashtags: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- text: string (nullable = true)\n",
      " |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |-- symbols: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- text: string (nullable = true)\n",
      " |    |    |-- urls: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |-- user_mentions: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- name: string (nullable = true)\n",
      " |    |    |    |    |-- screen_name: string (nullable = true)\n",
      " |    |-- extended_entities: struct (nullable = true)\n",
      " |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |    |-- video_info: struct (nullable = true)\n",
      " |    |    |    |    |    |-- aspect_ratio: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- duration_millis: long (nullable = true)\n",
      " |    |    |    |    |    |-- variants: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |    |-- bitrate: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- content_type: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |-- extended_tweet: struct (nullable = true)\n",
      " |    |    |-- display_text_range: array (nullable = true)\n",
      " |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |-- entities: struct (nullable = true)\n",
      " |    |    |    |-- hashtags: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- text: string (nullable = true)\n",
      " |    |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |    |    |-- video_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- aspect_ratio: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |    |-- duration_millis: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- variants: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |    |    |-- bitrate: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- content_type: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |-- symbols: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- text: string (nullable = true)\n",
      " |    |    |    |-- urls: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |-- user_mentions: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- name: string (nullable = true)\n",
      " |    |    |    |    |    |-- screen_name: string (nullable = true)\n",
      " |    |    |-- extended_entities: struct (nullable = true)\n",
      " |    |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |    |    |-- video_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- aspect_ratio: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |    |-- duration_millis: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- variants: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |    |    |-- bitrate: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- content_type: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |-- full_text: string (nullable = true)\n",
      " |    |-- favorite_count: long (nullable = true)\n",
      " |    |-- favorited: boolean (nullable = true)\n",
      " |    |-- filter_level: string (nullable = true)\n",
      " |    |-- geo: struct (nullable = true)\n",
      " |    |    |-- coordinates: array (nullable = true)\n",
      " |    |    |    |-- element: double (containsNull = true)\n",
      " |    |    |-- type: string (nullable = true)\n",
      " |    |-- id: long (nullable = true)\n",
      " |    |-- id_str: string (nullable = true)\n",
      " |    |-- in_reply_to_screen_name: string (nullable = true)\n",
      " |    |-- in_reply_to_status_id: long (nullable = true)\n",
      " |    |-- in_reply_to_status_id_str: string (nullable = true)\n",
      " |    |-- in_reply_to_user_id: long (nullable = true)\n",
      " |    |-- in_reply_to_user_id_str: string (nullable = true)\n",
      " |    |-- is_quote_status: boolean (nullable = true)\n",
      " |    |-- lang: string (nullable = true)\n",
      " |    |-- place: struct (nullable = true)\n",
      " |    |    |-- bounding_box: struct (nullable = true)\n",
      " |    |    |    |-- coordinates: array (nullable = true)\n",
      " |    |    |    |    |-- element: array (containsNull = true)\n",
      " |    |    |    |    |    |-- element: array (containsNull = true)\n",
      " |    |    |    |    |    |    |-- element: double (containsNull = true)\n",
      " |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |-- country: string (nullable = true)\n",
      " |    |    |-- country_code: string (nullable = true)\n",
      " |    |    |-- full_name: string (nullable = true)\n",
      " |    |    |-- id: string (nullable = true)\n",
      " |    |    |-- name: string (nullable = true)\n",
      " |    |    |-- place_type: string (nullable = true)\n",
      " |    |    |-- url: string (nullable = true)\n",
      " |    |-- possibly_sensitive: boolean (nullable = true)\n",
      " |    |-- quote_count: long (nullable = true)\n",
      " |    |-- quoted_status: struct (nullable = true)\n",
      " |    |    |-- coordinates: struct (nullable = true)\n",
      " |    |    |    |-- coordinates: array (nullable = true)\n",
      " |    |    |    |    |-- element: double (containsNull = true)\n",
      " |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |-- created_at: string (nullable = true)\n",
      " |    |    |-- display_text_range: array (nullable = true)\n",
      " |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |-- entities: struct (nullable = true)\n",
      " |    |    |    |-- hashtags: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- text: string (nullable = true)\n",
      " |    |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |-- symbols: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- text: string (nullable = true)\n",
      " |    |    |    |-- urls: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |-- user_mentions: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- name: string (nullable = true)\n",
      " |    |    |    |    |    |-- screen_name: string (nullable = true)\n",
      " |    |    |-- extended_entities: struct (nullable = true)\n",
      " |    |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |    |    |-- video_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |-- aspect_ratio: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |    |-- duration_millis: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- variants: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |    |    |-- bitrate: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- content_type: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |-- extended_tweet: struct (nullable = true)\n",
      " |    |    |    |-- display_text_range: array (nullable = true)\n",
      " |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |-- entities: struct (nullable = true)\n",
      " |    |    |    |    |-- hashtags: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |    |-- text: string (nullable = true)\n",
      " |    |    |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- video_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- aspect_ratio: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |    |    |-- duration_millis: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- variants: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |    |    |    |-- bitrate: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |    |-- content_type: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |    |-- symbols: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |    |-- text: string (nullable = true)\n",
      " |    |    |    |    |-- urls: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |    |-- user_mentions: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |    |-- name: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- screen_name: string (nullable = true)\n",
      " |    |    |    |-- extended_entities: struct (nullable = true)\n",
      " |    |    |    |    |-- media: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |-- additional_media_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- embeddable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- monetizable: boolean (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- title: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- display_url: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- expanded_url: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- indices: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |    |-- media_url: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- media_url_https: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- sizes: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- large: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- medium: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- small: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- thumb: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- h: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- resize: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- w: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- source_status_id: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- source_status_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- source_user_id: long (nullable = true)\n",
      " |    |    |    |    |    |    |-- source_user_id_str: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |    |    |    |-- video_info: struct (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- aspect_ratio: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- element: long (containsNull = true)\n",
      " |    |    |    |    |    |    |    |-- duration_millis: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |-- variants: array (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |-- element: struct (containsNull = true)\n",
      " |    |    |    |    |    |    |    |    |    |-- bitrate: long (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |    |-- content_type: string (nullable = true)\n",
      " |    |    |    |    |    |    |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |-- full_text: string (nullable = true)\n",
      " |    |    |-- favorite_count: long (nullable = true)\n",
      " |    |    |-- favorited: boolean (nullable = true)\n",
      " |    |    |-- filter_level: string (nullable = true)\n",
      " |    |    |-- geo: struct (nullable = true)\n",
      " |    |    |    |-- coordinates: array (nullable = true)\n",
      " |    |    |    |    |-- element: double (containsNull = true)\n",
      " |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |-- id: long (nullable = true)\n",
      " |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |-- in_reply_to_screen_name: string (nullable = true)\n",
      " |    |    |-- in_reply_to_status_id: long (nullable = true)\n",
      " |    |    |-- in_reply_to_status_id_str: string (nullable = true)\n",
      " |    |    |-- in_reply_to_user_id: long (nullable = true)\n",
      " |    |    |-- in_reply_to_user_id_str: string (nullable = true)\n",
      " |    |    |-- is_quote_status: boolean (nullable = true)\n",
      " |    |    |-- lang: string (nullable = true)\n",
      " |    |    |-- place: struct (nullable = true)\n",
      " |    |    |    |-- bounding_box: struct (nullable = true)\n",
      " |    |    |    |    |-- coordinates: array (nullable = true)\n",
      " |    |    |    |    |    |-- element: array (containsNull = true)\n",
      " |    |    |    |    |    |    |-- element: array (containsNull = true)\n",
      " |    |    |    |    |    |    |    |-- element: double (containsNull = true)\n",
      " |    |    |    |    |-- type: string (nullable = true)\n",
      " |    |    |    |-- country: string (nullable = true)\n",
      " |    |    |    |-- country_code: string (nullable = true)\n",
      " |    |    |    |-- full_name: string (nullable = true)\n",
      " |    |    |    |-- id: string (nullable = true)\n",
      " |    |    |    |-- name: string (nullable = true)\n",
      " |    |    |    |-- place_type: string (nullable = true)\n",
      " |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |-- possibly_sensitive: boolean (nullable = true)\n",
      " |    |    |-- quote_count: long (nullable = true)\n",
      " |    |    |-- quoted_status_id: long (nullable = true)\n",
      " |    |    |-- quoted_status_id_str: string (nullable = true)\n",
      " |    |    |-- reply_count: long (nullable = true)\n",
      " |    |    |-- retweet_count: long (nullable = true)\n",
      " |    |    |-- retweeted: boolean (nullable = true)\n",
      " |    |    |-- scopes: struct (nullable = true)\n",
      " |    |    |    |-- followers: boolean (nullable = true)\n",
      " |    |    |-- source: string (nullable = true)\n",
      " |    |    |-- text: string (nullable = true)\n",
      " |    |    |-- truncated: boolean (nullable = true)\n",
      " |    |    |-- user: struct (nullable = true)\n",
      " |    |    |    |-- contributors_enabled: boolean (nullable = true)\n",
      " |    |    |    |-- created_at: string (nullable = true)\n",
      " |    |    |    |-- default_profile: boolean (nullable = true)\n",
      " |    |    |    |-- default_profile_image: boolean (nullable = true)\n",
      " |    |    |    |-- description: string (nullable = true)\n",
      " |    |    |    |-- favourites_count: long (nullable = true)\n",
      " |    |    |    |-- followers_count: long (nullable = true)\n",
      " |    |    |    |-- friends_count: long (nullable = true)\n",
      " |    |    |    |-- geo_enabled: boolean (nullable = true)\n",
      " |    |    |    |-- id: long (nullable = true)\n",
      " |    |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |    |-- is_translator: boolean (nullable = true)\n",
      " |    |    |    |-- listed_count: long (nullable = true)\n",
      " |    |    |    |-- location: string (nullable = true)\n",
      " |    |    |    |-- name: string (nullable = true)\n",
      " |    |    |    |-- profile_background_color: string (nullable = true)\n",
      " |    |    |    |-- profile_background_image_url: string (nullable = true)\n",
      " |    |    |    |-- profile_background_image_url_https: string (nullable = true)\n",
      " |    |    |    |-- profile_background_tile: boolean (nullable = true)\n",
      " |    |    |    |-- profile_banner_url: string (nullable = true)\n",
      " |    |    |    |-- profile_image_url: string (nullable = true)\n",
      " |    |    |    |-- profile_image_url_https: string (nullable = true)\n",
      " |    |    |    |-- profile_link_color: string (nullable = true)\n",
      " |    |    |    |-- profile_sidebar_border_color: string (nullable = true)\n",
      " |    |    |    |-- profile_sidebar_fill_color: string (nullable = true)\n",
      " |    |    |    |-- profile_text_color: string (nullable = true)\n",
      " |    |    |    |-- profile_use_background_image: boolean (nullable = true)\n",
      " |    |    |    |-- protected: boolean (nullable = true)\n",
      " |    |    |    |-- screen_name: string (nullable = true)\n",
      " |    |    |    |-- statuses_count: long (nullable = true)\n",
      " |    |    |    |-- translator_type: string (nullable = true)\n",
      " |    |    |    |-- url: string (nullable = true)\n",
      " |    |    |    |-- verified: boolean (nullable = true)\n",
      " |    |    |    |-- verified_type: string (nullable = true)\n",
      " |    |    |    |-- withheld_in_countries: array (nullable = true)\n",
      " |    |    |    |    |-- element: string (containsNull = true)\n",
      " |    |    |-- withheld_in_countries: array (nullable = true)\n",
      " |    |    |    |-- element: string (containsNull = true)\n",
      " |    |-- quoted_status_id: long (nullable = true)\n",
      " |    |-- quoted_status_id_str: string (nullable = true)\n",
      " |    |-- quoted_status_permalink: struct (nullable = true)\n",
      " |    |    |-- display: string (nullable = true)\n",
      " |    |    |-- expanded: string (nullable = true)\n",
      " |    |    |-- url: string (nullable = true)\n",
      " |    |-- reply_count: long (nullable = true)\n",
      " |    |-- retweet_count: long (nullable = true)\n",
      " |    |-- retweeted: boolean (nullable = true)\n",
      " |    |-- scopes: struct (nullable = true)\n",
      " |    |    |-- followers: boolean (nullable = true)\n",
      " |    |    |-- place_ids: array (nullable = true)\n",
      " |    |    |    |-- element: string (containsNull = true)\n",
      " |    |-- source: string (nullable = true)\n",
      " |    |-- text: string (nullable = true)\n",
      " |    |-- truncated: boolean (nullable = true)\n",
      " |    |-- user: struct (nullable = true)\n",
      " |    |    |-- contributors_enabled: boolean (nullable = true)\n",
      " |    |    |-- created_at: string (nullable = true)\n",
      " |    |    |-- default_profile: boolean (nullable = true)\n",
      " |    |    |-- default_profile_image: boolean (nullable = true)\n",
      " |    |    |-- description: string (nullable = true)\n",
      " |    |    |-- favourites_count: long (nullable = true)\n",
      " |    |    |-- followers_count: long (nullable = true)\n",
      " |    |    |-- friends_count: long (nullable = true)\n",
      " |    |    |-- geo_enabled: boolean (nullable = true)\n",
      " |    |    |-- id: long (nullable = true)\n",
      " |    |    |-- id_str: string (nullable = true)\n",
      " |    |    |-- is_translator: boolean (nullable = true)\n",
      " |    |    |-- listed_count: long (nullable = true)\n",
      " |    |    |-- location: string (nullable = true)\n",
      " |    |    |-- name: string (nullable = true)\n",
      " |    |    |-- profile_background_color: string (nullable = true)\n",
      " |    |    |-- profile_background_image_url: string (nullable = true)\n",
      " |    |    |-- profile_background_image_url_https: string (nullable = true)\n",
      " |    |    |-- profile_background_tile: boolean (nullable = true)\n",
      " |    |    |-- profile_banner_url: string (nullable = true)\n",
      " |    |    |-- profile_image_url: string (nullable = true)\n",
      " |    |    |-- profile_image_url_https: string (nullable = true)\n",
      " |    |    |-- profile_link_color: string (nullable = true)\n",
      " |    |    |-- profile_sidebar_border_color: string (nullable = true)\n",
      " |    |    |-- profile_sidebar_fill_color: string (nullable = true)\n",
      " |    |    |-- profile_text_color: string (nullable = true)\n",
      " |    |    |-- profile_use_background_image: boolean (nullable = true)\n",
      " |    |    |-- protected: boolean (nullable = true)\n",
      " |    |    |-- screen_name: string (nullable = true)\n",
      " |    |    |-- statuses_count: long (nullable = true)\n",
      " |    |    |-- translator_type: string (nullable = true)\n",
      " |    |    |-- url: string (nullable = true)\n",
      " |    |    |-- verified: boolean (nullable = true)\n",
      " |    |    |-- verified_type: string (nullable = true)\n",
      " |    |    |-- withheld_in_countries: array (nullable = true)\n",
      " |    |    |    |-- element: string (containsNull = true)\n",
      " |    |-- withheld_in_countries: array (nullable = true)\n",
      " |    |    |-- element: string (containsNull = true)\n",
      " |-- source: string (nullable = true)\n",
      " |-- text: string (nullable = true)\n",
      " |-- timestamp_ms: string (nullable = true)\n",
      " |-- truncated: boolean (nullable = true)\n",
      " |-- tweet_text: string (nullable = true)\n",
      " |-- user: struct (nullable = true)\n",
      " |    |-- contributors_enabled: boolean (nullable = true)\n",
      " |    |-- created_at: string (nullable = true)\n",
      " |    |-- default_profile: boolean (nullable = true)\n",
      " |    |-- default_profile_image: boolean (nullable = true)\n",
      " |    |-- description: string (nullable = true)\n",
      " |    |-- favourites_count: long (nullable = true)\n",
      " |    |-- followers_count: long (nullable = true)\n",
      " |    |-- friends_count: long (nullable = true)\n",
      " |    |-- geo_enabled: boolean (nullable = true)\n",
      " |    |-- id: long (nullable = true)\n",
      " |    |-- id_str: string (nullable = true)\n",
      " |    |-- is_translator: boolean (nullable = true)\n",
      " |    |-- listed_count: long (nullable = true)\n",
      " |    |-- location: string (nullable = true)\n",
      " |    |-- name: string (nullable = true)\n",
      " |    |-- profile_background_color: string (nullable = true)\n",
      " |    |-- profile_background_image_url: string (nullable = true)\n",
      " |    |-- profile_background_image_url_https: string (nullable = true)\n",
      " |    |-- profile_background_tile: boolean (nullable = true)\n",
      " |    |-- profile_banner_url: string (nullable = true)\n",
      " |    |-- profile_image_url: string (nullable = true)\n",
      " |    |-- profile_image_url_https: string (nullable = true)\n",
      " |    |-- profile_link_color: string (nullable = true)\n",
      " |    |-- profile_sidebar_border_color: string (nullable = true)\n",
      " |    |-- profile_sidebar_fill_color: string (nullable = true)\n",
      " |    |-- profile_text_color: string (nullable = true)\n",
      " |    |-- profile_use_background_image: boolean (nullable = true)\n",
      " |    |-- protected: boolean (nullable = true)\n",
      " |    |-- screen_name: string (nullable = true)\n",
      " |    |-- statuses_count: long (nullable = true)\n",
      " |    |-- translator_type: string (nullable = true)\n",
      " |    |-- url: string (nullable = true)\n",
      " |    |-- verified: boolean (nullable = true)\n",
      " |    |-- verified_type: string (nullable = true)\n",
      " |    |-- withheld_in_countries: array (nullable = true)\n",
      " |    |    |-- element: string (containsNull = true)\n",
      " |-- withheld_in_countries: array (nullable = true)\n",
      " |    |-- element: string (containsNull = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tweets_df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "cfdcf52a-2343-4ee0-b812-3e359ceec570",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "99994342"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Checking for the total number of records\n",
    "total_recs = tweets_df.count()\n",
    "total_recs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9c333531-a0b3-43db-976c-177ce4b71839",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "39"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Checking how many columns are present to proceed with EDA\n",
    "total_features = len(tweets_df.columns)\n",
    "total_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "44f5e0f9-3ea2-4657-8a91-444286d25ef3",
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_df = tweets_df\\\n",
    ".withColumn('tweet_text', lower('tweet_text'))\\\n",
    ".withColumn('stripped', regexp_replace(col(\"tweet_text\"),\"[\\$#,&%\\\".]\",\"\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "047563d7-480c-46c1-b46d-ae39597c26d3",
   "metadata": {},
   "source": [
    "# Filtering the text and hashtag columns to get the tweets only related to education (helps in data filtration)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "5c66353a-4de8-4d81-b4c2-4e3705b20a76",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 21:==========>                                        (1191 + 32) / 5741]\r"
     ]
    }
   ],
   "source": [
    "tweets_df = tweets_df.filter(tweets_df.lang == 'en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "15392e6d-25bd-435a-8d39-20d319f4f708",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 21:===========>                                       (1273 + 32) / 5741]\r"
     ]
    }
   ],
   "source": [
    "tdf_tweet_text = tweets_df.filter(col('tweet_text').rlike('.*tuition.*|.*Tuition.*|.*education.*|.*Education.*|.*school.*|.*School.*|.*college.*|.*College.*|.*university.*|.*University.*|.*grade.*|.*Grade.*'))\n",
    "tdf_tweet_text.createOrReplaceTempView(\"tw_df_tweet_text\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "9c7b9ff9-07d2-4874-a96c-9885cf1d88b4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 21:===========>                                       (1316 + 32) / 5741]\r"
     ]
    }
   ],
   "source": [
    "tdf_hashtag = spark.sql('select * from tw_df_tweet_text where exists(entities.hashtags.text, x -> x rlike \".*education.*|.*Education.*|.*school.*|.*School.*|.*college.*|.*College.*|.*university.*|.*University.*|.*grade.*|.*Grade.*|.*K12.*|.*k12.*|.*study.*|.*Study.*|.*edchat.*|.*Edchat.*|.*learning.*|.*Learning.*|.*edreform.*|.*Edreform.*|.*scichat.*|.*student.*|.*Student.*\")')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8908f0e3-280e-44a6-91ab-4b1fe15da85c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "tdf_hashtag.write.format(\"parquet\").\\\n",
    "mode('overwrite').\\\n",
    "save('gs://' + 'msca-bdp-students-bucket/shared_data/kishorkumarreddy' + '/filtered_data_for_analysis_1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94064b22-9a02-44b6-b671-402bf6abc0ca",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1005167"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tdf_hashtag.count()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}