spark-geomesa-hbase.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Apache Spark with GeoMesa and Apache HBase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pyspark\n",
    "from pyspark import SparkConf\n",
    "from pyspark import SparkContext\n",
    "from pyspark.sql import SparkSession\n",
    "\n",
    "import geomesa_pyspark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "geomesa_fs_jar_filepath = os.path.join(\n",
    "    os.environ['GEOMESA_FS_HOME'], \n",
    "    \"dist/spark/geomesa-fs-spark-runtime_2.11-3.0.0.jar\")\n",
    "geomesa_hbase_jar_filepath = os.path.join(\n",
    "    os.environ['GEOMESA_HBASE_HOME'],\n",
    "    \"dist/spark/geomesa-hbase-spark-runtime-hbase2_2.11-3.0.0.jar\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "conf = geomesa_pyspark.configure(\n",
    "    jars=[geomesa_hbase_jar_filepath],\n",
    "    packages=['geomesa_pyspark','pytz'],\n",
    "    spark_home=os.environ['SPARK_HOME']).\\\n",
    "    setAppName('geomesa_hbase')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 48.9 ms, sys: 21.8 ms, total: 70.7 ms\n",
      "Wall time: 30.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "spark = SparkSession \\\n",
    "    .builder \\\n",
    "    .config(conf=conf) \\\n",
    "    .enableHiveSupport() \\\n",
    "    .getOrCreate()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare DataFrames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 9.78 ms, sys: 4.66 ms, total: 14.4 ms\n",
      "Wall time: 50 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "86770"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "params = {\"hbase.catalog\": \"gdelt_custom_20200101\"}\n",
    "\n",
    "df_hbase = spark.read.format(\"geomesa\") \\\n",
    "                .options(**params) \\\n",
    "                .option(\"geomesa.feature\", \"gdelt_custom\") \\\n",
    "                .load()\n",
    "\n",
    "df_hbase.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.99 ms, sys: 0 ns, total: 3.99 ms\n",
      "Wall time: 2.42 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "176"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "params = {\"hbase.catalog\": \"ne_countries\"}\n",
    "\n",
    "df_hbase_ne = spark.read.format(\"geomesa\") \\\n",
    "                   .options(**params) \\\n",
    "                   .option(\"geomesa.feature\", \"ne_countries\") \\\n",
    "                   .load()\n",
    "\n",
    "df_hbase_ne.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------------+-----------+\n",
      "|database|           tableName|isTemporary|\n",
      "+--------+--------------------+-----------+\n",
      "| default|      gdelt_csv_2019|      false|\n",
      "| default|      gdelt_csv_2020|      false|\n",
      "| default|  gdelt_parquet_2020|      false|\n",
      "| default|gdelt_parquet_ins...|      false|\n",
      "| default|gdelt_parquet_ins...|      false|\n",
      "| default|ne_10_states_prov...|      false|\n",
      "| default|ne_110_countries_...|      false|\n",
      "|        |gdelt_custom_2020...|       true|\n",
      "|        |        ne_countries|       true|\n",
      "+--------+--------------------+-----------+\n",
      "\n",
      "CPU times: user 29.9 ms, sys: 14.9 ms, total: 44.7 ms\n",
      "Wall time: 3min 1s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df_hbase.createOrReplaceTempView(\"gdelt_custom_20200101\")\n",
    "df_hbase_ne.createOrReplaceTempView(\"ne_countries\")\n",
    "spark.sql(\"SHOW TABLES\").show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Geospatial Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "EXTENT = [9.5307, 46.3723, 17.1608, 49.0205]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 479 ms, sys: 241 ms, total: 720 ms\n",
      "Wall time: 2min 8s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>event_root_code</th>\n",
       "      <th>cnt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   event_root_code  cnt\n",
       "0                4   72\n",
       "1                1   39\n",
       "2                5   31\n",
       "3                3   18\n",
       "4                2   13"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "df = spark.sql(\"\"\"\n",
    "  SELECT\n",
    "    event_root_code,\n",
    "    COUNT(event_id) AS cnt\n",
    "  FROM\n",
    "    gdelt_custom_20200101\n",
    "  WHERE \n",
    "    ST_Within(geom, st_makeBBOX({}, {}, {}, {}))\n",
    "  GROUP BY event_root_code\n",
    "  ORDER BY cnt DESC\n",
    "\"\"\".format(*EXTENT)).toPandas()\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Spatial Join"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 30.9 ms, sys: 10.6 ms, total: 41.5 ms\n",
      "Wall time: 59.2 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>iso_a2</th>\n",
       "      <th>cnt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>MM</td>\n",
       "      <td>65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DZ</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LT</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CI</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AZ</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  iso_a2  cnt\n",
       "0     MM   65\n",
       "1     DZ   56\n",
       "2     LT   30\n",
       "3     CI   47\n",
       "4     AZ   48"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "df = spark.sql(\"\"\"\n",
    "  SELECT\n",
    "    c.iso_a2,\n",
    "    COUNT(g.event_id) AS cnt\n",
    "  FROM\n",
    "    gdelt_custom_20200101 AS g,\n",
    "    ne_countries AS c\n",
    "  WHERE ST_Within(g.geom, c.polygons)\n",
    "  GROUP BY c.iso_a2\n",
    "\"\"\").toPandas()\n",
    "\n",
    "df.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "spark",
   "language": "python",
   "name": "spark"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}