milestone2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "emotional-emperor",
   "metadata": {},
   "source": [
    "#### create BigQuery Dataset \"datamart\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "exceptional-multimedia",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset 'lunar-analyzer-302702:datamart' successfully created.\n"
     ]
    }
   ],
   "source": [
    "!bq --location=US mk --dataset datamart"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sound-usage",
   "metadata": {},
   "source": [
    "### Transform the COVID-19 Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "piano-threshold",
   "metadata": {},
   "source": [
    "#### create preliminary covid table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "understanding-product",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE OR REPLACE TABLE datamart.covid AS\n",
    "SELECT null AS id, fips, admin2, province_state, country_region AS Country, last_update, latitude, longitude, confirmed, deaths, recovered, active, combined_key, incident_rate, case_fatality_ratio\n",
    "FROM jhu_daily_reports_staging.covid_winter2020\n",
    "UNION ALL\n",
    "SELECT null AS id, fips, admin2, province_state, country_region AS Country, last_update, latitude, longitude, confirmed, deaths, recovered, active, combined_key, incident_rate, case_fatality_ratio\n",
    "FROM jhu_daily_reports_staging.covid_spring2020\n",
    "UNION ALL\n",
    "SELECT null AS id, fips, admin2, province_state, country_region AS Country, CAST(last_update AS STRING) AS last_update, lat AS latitude, long_ AS longitude, confirmed, deaths, recovered, active, combined_key, incidence_rate AS incident_rate, case_fatality_ratio\n",
    "FROM jhu_daily_reports_staging.covid_summer2020\n",
    "UNION ALL\n",
    "SELECT null AS id, fips, admin2, province_state, country_region AS Country, CAST(last_update AS STRING) AS last_update, lat AS latitude, long_ AS longitude, confirmed, deaths, recovered, active, combined_key, incident_rate, case_fatality_ratio\n",
    "FROM jhu_daily_reports_staging.covid_fall2020\n",
    "UNION ALL\n",
    "SELECT null AS id, fips, admin2, province_state, country_region AS Country, CAST(last_update AS STRING) AS last_update, lat AS latitude, long_ AS longitude, confirmed, deaths, recovered, active, combined_key, incident_rate, case_fatality_ratio\n",
    "FROM jhu_daily_reports_staging.covid_winter2021\n",
    "UNION ALL\n",
    "SELECT null AS id, fips, admin2, province_state, country_region AS Country, CAST(last_update AS STRING) AS last_update, lat AS latitude, long_ AS longitude, confirmed, deaths, recovered, active, combined_key, incident_rate, case_fatality_ratio\n",
    "FROM jhu_daily_reports_staging.covid_spring2021"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "destroyed-oracle",
   "metadata": {},
   "source": [
    "#### compute fingerprint ids for preliminary covid table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "introductory-custody",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.covid\n",
    "SET id=FARM_FINGERPRINT(country)\n",
    "WHERE province_state IS null"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "motivated-devices",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.covid\n",
    "SET id=FARM_FINGERPRINT(concat(province_state, country))\n",
    "WHERE (province_state IS NOT null AND country!='US')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "fundamental-dragon",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.covid\n",
    "SET id=FARM_FINGERPRINT(concat(admin2, province_state, country))\n",
    "WHERE (province_state IS NOT null AND country='US' AND admin2 IS NOT null AND fips IS NULL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "native-covering",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.covid\n",
    "SET id=FARM_FINGERPRINT(concat(CAST(fips AS STRING), admin2, province_state, country))\n",
    "WHERE (province_state IS NOT null AND country='US' AND admin2 IS NOT null AND fips IS NOT null)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "planned-stream",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.covid\n",
    "SET id=FARM_FINGERPRINT(concat(CAST(fips AS STRING), province_state, country))\n",
    "WHERE (province_state IS NOT null AND country='US' AND admin2 IS null AND fips IS NOT null)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "incoming-privacy",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.covid\n",
    "SET id=FARM_FINGERPRINT(concat(province_state, country))\n",
    "WHERE (province_state IS NOT null AND country='US' AND admin2 IS null AND fips IS null)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "municipal-triumph",
   "metadata": {},
   "source": [
    "#### check that there is no null id field in the preliminary covid table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "noted-popularity",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>null_id_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   null_id_count\n",
       "0              0"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT count(*) AS null_id_count\n",
    "FROM datamart.covid\n",
    "WHERE id IS null"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "informational-crime",
   "metadata": {},
   "source": [
    "#### sample preliminary covid table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "confirmed-selling",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>fips</th>\n",
       "      <th>admin2</th>\n",
       "      <th>province_state</th>\n",
       "      <th>Country</th>\n",
       "      <th>last_update</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>confirmed</th>\n",
       "      <th>deaths</th>\n",
       "      <th>recovered</th>\n",
       "      <th>active</th>\n",
       "      <th>combined_key</th>\n",
       "      <th>incident_rate</th>\n",
       "      <th>case_fatality_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8576431891811451300</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Azerbaijan</td>\n",
       "      <td>2020-02-28 15:03:26</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8778414404485170876</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>2020-10-13 04:24:02+00</td>\n",
       "      <td>33.93911</td>\n",
       "      <td>67.709953</td>\n",
       "      <td>39870</td>\n",
       "      <td>1481</td>\n",
       "      <td>33118</td>\n",
       "      <td>5273.0</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>102.418955</td>\n",
       "      <td>3.709556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8778414404485170876</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>2020-12-24 05:23:04+00</td>\n",
       "      <td>33.93911</td>\n",
       "      <td>67.709953</td>\n",
       "      <td>50433</td>\n",
       "      <td>2117</td>\n",
       "      <td>39692</td>\n",
       "      <td>8624.0</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>129.553427</td>\n",
       "      <td>4.197648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8778414404485170876</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>2020-09-05 04:28:19+00</td>\n",
       "      <td>33.93911</td>\n",
       "      <td>67.709953</td>\n",
       "      <td>38304</td>\n",
       "      <td>1410</td>\n",
       "      <td>29713</td>\n",
       "      <td>7182.0</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>98.396179</td>\n",
       "      <td>3.678467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8778414404485170876</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>2021-03-29 04:28:35+00</td>\n",
       "      <td>33.93911</td>\n",
       "      <td>67.709953</td>\n",
       "      <td>56294</td>\n",
       "      <td>2470</td>\n",
       "      <td>50013</td>\n",
       "      <td>3811.0</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>144.609296</td>\n",
       "      <td>4.387679</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    id  fips admin2 province_state      Country  \\\n",
       "0  8576431891811451300   NaN   None           None   Azerbaijan   \n",
       "1  8778414404485170876   NaN   None           None  Afghanistan   \n",
       "2  8778414404485170876   NaN   None           None  Afghanistan   \n",
       "3  8778414404485170876   NaN   None           None  Afghanistan   \n",
       "4  8778414404485170876   NaN   None           None  Afghanistan   \n",
       "\n",
       "              last_update  latitude  longitude  confirmed  deaths  recovered  \\\n",
       "0     2020-02-28 15:03:26       NaN        NaN          1       0          0   \n",
       "1  2020-10-13 04:24:02+00  33.93911  67.709953      39870    1481      33118   \n",
       "2  2020-12-24 05:23:04+00  33.93911  67.709953      50433    2117      39692   \n",
       "3  2020-09-05 04:28:19+00  33.93911  67.709953      38304    1410      29713   \n",
       "4  2021-03-29 04:28:35+00  33.93911  67.709953      56294    2470      50013   \n",
       "\n",
       "   active combined_key  incident_rate  case_fatality_ratio  \n",
       "0     NaN         None            NaN                  NaN  \n",
       "1  5273.0  Afghanistan     102.418955             3.709556  \n",
       "2  8624.0  Afghanistan     129.553427             4.197648  \n",
       "3  7182.0  Afghanistan      98.396179             3.678467  \n",
       "4  3811.0  Afghanistan     144.609296             4.387679  "
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT *\n",
    "FROM datamart.covid\n",
    "ORDER BY province_state, country\n",
    "LIMIT 5"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "roman-overhead",
   "metadata": {},
   "source": [
    "### Transforms for table locations"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bottom-apparel",
   "metadata": {},
   "source": [
    "#### create locations_raw table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "facial-laugh",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE OR REPLACE TABLE datamart.Locations_raw AS\n",
    "SELECT DISTINCT id, fips, admin2, province_state, country, latitude, longitude, combined_key\n",
    "FROM datamart.covid"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "technological-plumbing",
   "metadata": {},
   "source": [
    "#### show rows that have city,state in province_state column of the locations_raw table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "confirmed-portuguese",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>fips</th>\n",
       "      <th>admin2</th>\n",
       "      <th>province_state</th>\n",
       "      <th>index</th>\n",
       "      <th>country</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>combined_key</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8145114016361068042</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>Tempe, AZ</td>\n",
       "      <td>6</td>\n",
       "      <td>US</td>\n",
       "      <td>33.4255</td>\n",
       "      <td>-111.9400</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8145114016361068042</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>Tempe, AZ</td>\n",
       "      <td>6</td>\n",
       "      <td>US</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8556287708025666455</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>Boston, MA</td>\n",
       "      <td>7</td>\n",
       "      <td>US</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8556287708025666455</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>Boston, MA</td>\n",
       "      <td>7</td>\n",
       "      <td>US</td>\n",
       "      <td>42.3601</td>\n",
       "      <td>-71.0589</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1673719292839792409</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>Orange, CA</td>\n",
       "      <td>7</td>\n",
       "      <td>US</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-1673719292839792409</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>Orange, CA</td>\n",
       "      <td>7</td>\n",
       "      <td>US</td>\n",
       "      <td>33.7879</td>\n",
       "      <td>-117.8531</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7702389166849437576</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>Travis, CA</td>\n",
       "      <td>7</td>\n",
       "      <td>US</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2904588917003491558</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>Ashland, NE</td>\n",
       "      <td>8</td>\n",
       "      <td>US</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8162844378884388227</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>Chicago, IL</td>\n",
       "      <td>8</td>\n",
       "      <td>US</td>\n",
       "      <td>41.8781</td>\n",
       "      <td>-87.6298</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>8162844378884388227</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>Chicago, IL</td>\n",
       "      <td>8</td>\n",
       "      <td>US</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    id  fips admin2 province_state  index country  latitude  \\\n",
       "0  8145114016361068042   NaN   None      Tempe, AZ      6      US   33.4255   \n",
       "1  8145114016361068042   NaN   None      Tempe, AZ      6      US       NaN   \n",
       "2  8556287708025666455   NaN   None     Boston, MA      7      US       NaN   \n",
       "3  8556287708025666455   NaN   None     Boston, MA      7      US   42.3601   \n",
       "4 -1673719292839792409   NaN   None     Orange, CA      7      US       NaN   \n",
       "5 -1673719292839792409   NaN   None     Orange, CA      7      US   33.7879   \n",
       "6  7702389166849437576   NaN   None     Travis, CA      7      US       NaN   \n",
       "7  2904588917003491558   NaN   None    Ashland, NE      8      US       NaN   \n",
       "8  8162844378884388227   NaN   None    Chicago, IL      8      US   41.8781   \n",
       "9  8162844378884388227   NaN   None    Chicago, IL      8      US       NaN   \n",
       "\n",
       "   longitude combined_key  \n",
       "0  -111.9400         None  \n",
       "1        NaN         None  \n",
       "2        NaN         None  \n",
       "3   -71.0589         None  \n",
       "4        NaN         None  \n",
       "5  -117.8531         None  \n",
       "6        NaN         None  \n",
       "7        NaN         None  \n",
       "8   -87.6298         None  \n",
       "9        NaN         None  "
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT id, fips, admin2, province_state, strpos(province_state, ',') AS index, country, latitude, longitude, combined_key\n",
    "FROM datamart.Locations_raw\n",
    "WHERE strpos(province_state, ',') > 0\n",
    "LIMIT 10"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "national-substance",
   "metadata": {},
   "source": [
    "#### standardize city, state in the locations_raw table into table locations_city"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "under-buffer",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE OR REPLACE TABLE datamart.Locations_city AS\n",
    "SELECT id, fips, admin2, split(province_state, ',')[offset(0)] AS city, split(province_state, ',')[offset(1)] AS state, country, latitude, longitude, combined_key\n",
    "FROM datamart.Locations_raw\n",
    "WHERE strpos(province_state, ',') > 0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "terminal-success",
   "metadata": {},
   "source": [
    "#### create locations table (if error, try CAST(null AS string) AS City"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "mexican-offer",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE OR REPLACE TABLE datamart.locations AS\n",
    "(SELECT id, fips, admin2, city, state, country, latitude, longitude, combined_key\n",
    "FROM datamart.Locations_city\n",
    "UNION ALL\n",
    "SELECT id, fips, admin2, null AS City, province_state AS State, country, latitude, longitude, combined_key\n",
    "FROM datamart.Locations_raw\n",
    "WHERE strpos(province_state, ',') = 0\n",
    "UNION ALL\n",
    "SELECT id, fips, admin2, null AS City, province_state AS State, country, latitude, longitude, combined_key\n",
    "FROM datamart.Locations_raw\n",
    "WHERE province_state IS null\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "joint-colony",
   "metadata": {},
   "source": [
    "#### Ensure no rows were lost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "numeric-chest",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>locations_raw_count</th>\n",
       "      <th>locations_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5637</td>\n",
       "      <td>5637</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   locations_raw_count  locations_count\n",
       "0                 5637             5637"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT count(*) AS locations_raw_count, (SELECT count(*) FROM datamart.locations) AS locations_count\n",
    "FROM datamart.Locations_raw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "professional-oriental",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE OR REPLACE TABLE datamart.locations AS\n",
    "SELECT id, fips, admin2, city, state, country, latitude, longitude, combined_key\n",
    "FROM\n",
    "    (SELECT *, rank() over (partition by id, country ORDER BY latitude DESC, longitude DESC, combined_key DESC) AS rank\n",
    "    FROM datamart.locations)\n",
    "WHERE rank = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "seventh-connectivity",
   "metadata": {},
   "source": [
    "#### PK check: if same number then no duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "extended-doctor",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>distinct_PK</th>\n",
       "      <th>locations_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4413</td>\n",
       "      <td>4413</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   distinct_PK  locations_count\n",
       "0         4413             4413"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT count(*) AS distinct_PK, (SELECT count(*) FROM datamart.locations) AS locations_count\n",
    "FROM (\n",
    "    SELECT DISTINCT id\n",
    "    FROM datamart.locations\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "alive-pharmacy",
   "metadata": {},
   "source": [
    "### Transforms for table cases"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "enhanced-rebecca",
   "metadata": {},
   "source": [
    "#### create Cases_raw table from preliminary covid table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "descending-karma",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE OR REPLACE TABLE datamart.Cases_raw AS\n",
    "SELECT id AS location_id, last_update, confirmed, deaths, recovered, active, incident_rate, case_fatality_ratio\n",
    "FROM datamart.covid"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "delayed-prediction",
   "metadata": {},
   "source": [
    "#### sample distinct dates in cases_raw table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "established-chemistry",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>last_update</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-12-23 05:22:03+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021-02-22 05:24:21+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020-10-16 04:24:01+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021-03-03 05:23:28+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020-11-14 05:25:37+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2020-08-14 04:51:19+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2020-09-25 04:23:00+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2020-07-06 04:33:57+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2020-10-28 04:24:39+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2021-01-30 05:22:49+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              last_update\n",
       "0  2020-12-23 05:22:03+00\n",
       "1  2021-02-22 05:24:21+00\n",
       "2  2020-10-16 04:24:01+00\n",
       "3  2021-03-03 05:23:28+00\n",
       "4  2020-11-14 05:25:37+00\n",
       "5  2020-08-14 04:51:19+00\n",
       "6  2020-09-25 04:23:00+00\n",
       "7  2020-07-06 04:33:57+00\n",
       "8  2020-10-28 04:24:39+00\n",
       "9  2021-01-30 05:22:49+00"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT DISTINCT last_update\n",
    "FROM datamart.Cases_raw\n",
    "LIMIT 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "exposed-sample",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>last_update</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1/22/20 17:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1/23/20 17:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1/24/20 17:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1/25/20 17:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1/26/20 16:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1/27/20 23:59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1/28/20 23:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1/29/20 19:30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1/30/20 16:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1/31/20 10:37</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     last_update\n",
       "0  1/22/20 17:00\n",
       "1  1/23/20 17:00\n",
       "2  1/24/20 17:00\n",
       "3  1/25/20 17:00\n",
       "4  1/26/20 16:00\n",
       "5  1/27/20 23:59\n",
       "6  1/28/20 23:00\n",
       "7  1/29/20 19:30\n",
       "8  1/30/20 16:00\n",
       "9  1/31/20 10:37"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT DISTINCT last_update\n",
    "FROM datamart.Cases_raw\n",
    "ORDER BY last_update\n",
    "LIMIT 10"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "alternate-appointment",
   "metadata": {},
   "source": [
    "#### create Cases_hypen_date_fixed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "suited-establishment",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>last_update</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-03-15 18:20:18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-03-18 13:13:15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020-03-20 20:13:15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020-03-11 20:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020-02-07 23:43:02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2020-03-14 11:33:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2020-03-14 20:13:16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2020-02-29 12:13:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2020-02-06 23:23:02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2020-03-16 19:13:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2020-03-20 07:43:02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2020-03-02 15:03:23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>2020-03-18 14:53:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2020-03-21 17:43:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2020-03-18 14:13:56</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            last_update\n",
       "0   2020-03-15 18:20:18\n",
       "1   2020-03-18 13:13:15\n",
       "2   2020-03-20 20:13:15\n",
       "3   2020-03-11 20:00:00\n",
       "4   2020-02-07 23:43:02\n",
       "5   2020-03-14 11:33:06\n",
       "6   2020-03-14 20:13:16\n",
       "7   2020-02-29 12:13:10\n",
       "8   2020-02-06 23:23:02\n",
       "9   2020-03-16 19:13:13\n",
       "10  2020-03-20 07:43:02\n",
       "11  2020-03-02 15:03:23\n",
       "12  2020-03-18 14:53:05\n",
       "13  2020-03-21 17:43:03\n",
       "14  2020-03-18 14:13:56"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT last_update\n",
    "FROM datamart.Cases_raw\n",
    "WHERE strpos(last_update, '-') > 0 AND strpos(last_update, ' ') > 0 AND strpos(last_update, '+') = 0\n",
    "LIMIT 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "civilian-objective",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE OR REPLACE TABLE datamart.Cases_hyphen_date_fixed AS\n",
    "SELECT location_id, CAST(last_update AS DATETIME) AS last_update, confirmed, deaths, recovered, active, incident_rate, case_fatality_ratio\n",
    "FROM datamart.Cases_raw\n",
    "WHERE strpos(last_update, '-') > 0 AND strpos(last_update, ' ') > 0 AND strpos(last_update, '+') = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "agreed-abraham",
   "metadata": {},
   "source": [
    "#### create Cases_plus_date_fixed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "steady-trauma",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>last_update</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-12-23 05:22:03+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2021-02-22 05:24:21+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020-10-16 04:24:01+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2021-03-03 05:23:28+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020-11-14 05:25:37+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2020-08-14 04:51:19+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2020-09-25 04:23:00+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2020-07-06 04:33:57+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2020-10-28 04:24:39+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2021-01-30 05:22:49+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2020-10-12 04:23:45+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2020-09-27 04:23:00+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>2021-04-09 04:21:13+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2021-04-02 04:20:36+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2021-03-03 05:23:28+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               last_update\n",
       "0   2020-12-23 05:22:03+00\n",
       "1   2021-02-22 05:24:21+00\n",
       "2   2020-10-16 04:24:01+00\n",
       "3   2021-03-03 05:23:28+00\n",
       "4   2020-11-14 05:25:37+00\n",
       "5   2020-08-14 04:51:19+00\n",
       "6   2020-09-25 04:23:00+00\n",
       "7   2020-07-06 04:33:57+00\n",
       "8   2020-10-28 04:24:39+00\n",
       "9   2021-01-30 05:22:49+00\n",
       "10  2020-10-12 04:23:45+00\n",
       "11  2020-09-27 04:23:00+00\n",
       "12  2021-04-09 04:21:13+00\n",
       "13  2021-04-02 04:20:36+00\n",
       "14  2021-03-03 05:23:28+00"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT last_update\n",
    "FROM datamart.Cases_raw\n",
    "WHERE strpos(last_update, '-') > 0 AND strpos(last_update, ' ') > 0 AND strpos(last_update, '+') > 0\n",
    "LIMIT 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "billion-colon",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE OR REPLACE TABLE datamart.Cases_plus_date_fixed AS\n",
    "SELECT location_id,CAST(CAST(last_update AS TIMESTAMP) AS DATETIME) AS last_update, confirmed, deaths, recovered, active, incident_rate, case_fatality_ratio\n",
    "FROM datamart.Cases_raw\n",
    "WHERE strpos(last_update, '-') > 0 AND strpos(last_update, ' ') > 0 AND strpos(last_update, '+') > 0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "previous-provider",
   "metadata": {},
   "source": [
    "#### create Cases_slash_date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "prepared-mumbai",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>last_update</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5/30/20 2:32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6/1/20 2:32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5/31/20 2:32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5/12/20 3:32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5/19/20 2:32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5/4/20 2:32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>4/17/20 23:30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>5/28/20 2:32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>5/20/20 2:32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>5/26/20 2:32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>5/8/20 2:32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>5/5/20 2:32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>4/5/20 2:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>4/20/20 23:36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>4/21/20 23:30</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      last_update\n",
       "0    5/30/20 2:32\n",
       "1     6/1/20 2:32\n",
       "2    5/31/20 2:32\n",
       "3    5/12/20 3:32\n",
       "4    5/19/20 2:32\n",
       "5     5/4/20 2:32\n",
       "6   4/17/20 23:30\n",
       "7    5/28/20 2:32\n",
       "8    5/20/20 2:32\n",
       "9    5/26/20 2:32\n",
       "10    5/8/20 2:32\n",
       "11    5/5/20 2:32\n",
       "12    4/5/20 2:06\n",
       "13  4/20/20 23:36\n",
       "14  4/21/20 23:30"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT DISTINCT last_update\n",
    "FROM datamart.Cases_raw\n",
    "WHERE strpos(last_update, '/') > 0\n",
    "LIMIT 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "banner-membrane",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE OR REPLACE TABLE datamart.Cases_slash_date AS\n",
    "SELECT * \n",
    "FROM datamart.Cases_raw\n",
    "WHERE strpos(last_update, '/') > 0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fundamental-reviewer",
   "metadata": {},
   "source": [
    "#### check Cases_slash_date for their year format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "emotional-order",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>last_update</th>\n",
       "      <th>year_num_digits</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1/22/20 17:00</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1/23/20 17:00</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1/24/20 17:00</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5/1/20 2:32</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5/2/20 2:32</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5/3/20 2:32</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>5/4/20 2:32</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>5/5/20 2:32</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>5/6/20 2:32</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>5/7/20 2:32</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     last_update  year_num_digits\n",
       "0  1/22/20 17:00                2\n",
       "1  1/23/20 17:00                2\n",
       "2  1/24/20 17:00                2\n",
       "3    5/1/20 2:32                2\n",
       "4    5/2/20 2:32                2\n",
       "5    5/3/20 2:32                2\n",
       "6    5/4/20 2:32                2\n",
       "7    5/5/20 2:32                2\n",
       "8    5/6/20 2:32                2\n",
       "9    5/7/20 2:32                2"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT last_update, length(split(split(last_update, ' ')[offset(0)], '/')[offset(2)]) AS year_num_digits\n",
    "FROM (\n",
    "    SELECT DISTINCT last_update\n",
    "    FROM datamart.Cases_slash_date)\n",
    "LIMIT 10"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "secondary-arbitration",
   "metadata": {},
   "source": [
    "#### create Cases_slash_date_fixed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "changing-pastor",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE OR REPLACE TABLE datamart.Cases_slash_date_fixed AS\n",
    "(SELECT location_id, PARSE_DATETIME('%m/%d/%y %H:%M', last_update) AS Last_Update, confirmed, deaths, recovered, active, incident_rate, case_fatality_ratio\n",
    "FROM datamart.Cases_slash_date \n",
    "WHERE length(split(split(last_update, ' ')[offset(0)], '/')[offset(2)]) = 2\n",
    "UNION ALL\n",
    "SELECT location_id, PARSE_DATETIME('%m/%d/%Y %H:%M', last_update) AS Last_Update, confirmed, deaths, recovered, active, incident_rate, case_fatality_ratio\n",
    "FROM datamart.Cases_slash_date \n",
    "WHERE length(split(split(last_update, ' ')[offset(0)], '/')[offset(2)]) = 4\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "reserved-tooth",
   "metadata": {},
   "source": [
    "#### sample Cases_slash_date_fixed for fixed dates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "understanding-wonder",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>last_update</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-01-22 17:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-01-23 17:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020-01-24 17:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020-01-25 17:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020-01-26 16:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2020-01-27 23:59:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2020-01-28 23:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2020-01-29 19:30:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2020-01-30 16:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2020-01-31 08:15:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          last_update\n",
       "0 2020-01-22 17:00:00\n",
       "1 2020-01-23 17:00:00\n",
       "2 2020-01-24 17:00:00\n",
       "3 2020-01-25 17:00:00\n",
       "4 2020-01-26 16:00:00\n",
       "5 2020-01-27 23:59:00\n",
       "6 2020-01-28 23:00:00\n",
       "7 2020-01-29 19:30:00\n",
       "8 2020-01-30 16:00:00\n",
       "9 2020-01-31 08:15:00"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT * \n",
    "FROM (\n",
    "    SELECT DISTINCT last_update\n",
    "    FROM datamart.Cases_slash_date_fixed)\n",
    "ORDER BY last_update\n",
    "LIMIT 10"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "transsexual-ottawa",
   "metadata": {},
   "source": [
    "#### create cases table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "excess-member",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE OR REPLACE TABLE datamart.cases AS\n",
    "(SELECT * \n",
    "FROM datamart.Cases_hyphen_date_fixed\n",
    "UNION ALL\n",
    "SELECT *\n",
    "FROM datamart.Cases_plus_date_fixed\n",
    "UNION ALL\n",
    "SELECT *\n",
    "FROM datamart.Cases_slash_date_fixed\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sticky-mainstream",
   "metadata": {},
   "source": [
    "#### ensure no rows were lost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "wrong-damage",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cases_raw_count</th>\n",
       "      <th>cases_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1657069</td>\n",
       "      <td>1657068</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   cases_raw_count  cases_count\n",
       "0          1657069      1657068"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT count(*) AS cases_raw_count, (SELECT count(*) FROM datamart.cases) AS cases_count\n",
    "FROM datamart.Cases_raw"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "turkish-cutting",
   "metadata": {},
   "source": [
    "#### remove duplicate cases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "prescription-event",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE OR REPLACE TABLE datamart.cases AS\n",
    "SELECT DISTINCT *\n",
    "FROM datamart.cases"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "current-owner",
   "metadata": {},
   "source": [
    "#### get cases count after first attempt of removing duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "intense-rainbow",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>distinct_cases_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1476173</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   distinct_cases_count\n",
       "0               1476173"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT count(*) AS distinct_cases_count\n",
    "FROM datamart.cases"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "polished-prerequisite",
   "metadata": {},
   "source": [
    "#### check if there a still duplicate cases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "entertaining-basin",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>location_id</th>\n",
       "      <th>last_update</th>\n",
       "      <th>duplicate_cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3183797238637395852</td>\n",
       "      <td>2021-04-02 15:13:53</td>\n",
       "      <td>146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1181716864917925688</td>\n",
       "      <td>2021-04-02 15:13:53</td>\n",
       "      <td>111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2231787937947570706</td>\n",
       "      <td>2020-12-21 13:27:30</td>\n",
       "      <td>92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8778414404485170876</td>\n",
       "      <td>2020-03-17 11:53:10</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8778414404485170876</td>\n",
       "      <td>2020-02-24 23:33:02</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-391570144277816991</td>\n",
       "      <td>2020-03-14 22:33:04</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>8009782315755301025</td>\n",
       "      <td>2020-08-04 02:27:56</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7612477503619687113</td>\n",
       "      <td>2020-02-13 17:53:03</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>7612477503619687113</td>\n",
       "      <td>2020-02-17 07:33:02</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>7430223413705102985</td>\n",
       "      <td>2020-03-14 20:13:16</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           location_id         last_update  duplicate_cases\n",
       "0  3183797238637395852 2021-04-02 15:13:53              146\n",
       "1 -1181716864917925688 2021-04-02 15:13:53              111\n",
       "2  2231787937947570706 2020-12-21 13:27:30               92\n",
       "3  8778414404485170876 2020-03-17 11:53:10                3\n",
       "4  8778414404485170876 2020-02-24 23:33:02                3\n",
       "5  -391570144277816991 2020-03-14 22:33:04                2\n",
       "6  8009782315755301025 2020-08-04 02:27:56                2\n",
       "7  7612477503619687113 2020-02-13 17:53:03                2\n",
       "8  7612477503619687113 2020-02-17 07:33:02                2\n",
       "9  7430223413705102985 2020-03-14 20:13:16                2"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT location_id, last_update, count(*) AS duplicate_cases\n",
    "FROM datamart.cases\n",
    "GROUP BY location_id, last_update\n",
    "HAVING count(*) > 1\n",
    "ORDER BY count(*) DESC\n",
    "LIMIT 10"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "backed-aspect",
   "metadata": {},
   "source": [
    "#### remove duplicates based on confirmed, deaths, and recovered cases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "theoretical-wildlife",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE OR REPLACE TABLE datamart.cases AS\n",
    "SELECT location_id, last_update, confirmed, deaths, recovered, active, incident_rate, case_fatality_ratio\n",
    "FROM(SELECT *, rank() over (partition by location_id, last_update order by confirmed DESC, deaths DESC, recovered DESC ) as rank\n",
    "FROM datamart.cases)\n",
    "WHERE rank = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "urban-probe",
   "metadata": {},
   "source": [
    "#### PK check: if same number then no duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "hollow-deployment",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>distinct_PK</th>\n",
       "      <th>cases_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1475767</td>\n",
       "      <td>1475767</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   distinct_PK  cases_count\n",
       "0      1475767      1475767"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT count(*) AS distinct_PK, (SELECT count(*) FROM datamart.cases) AS cases_count\n",
    "FROM (\n",
    "    SELECT DISTINCT location_id, last_update\n",
    "    FROM datamart.cases\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "adopted-recycling",
   "metadata": {},
   "source": [
    "#### FK check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "fancy-muslim",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>foreign_key_violations</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   foreign_key_violations\n",
       "0                       0"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT count(*) AS foreign_key_violations\n",
    "FROM datamart.cases AS c\n",
    "LEFT JOIN datamart.locations AS l ON c.location_id=l.id\n",
    "WHERE l.id IS null"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "architectural-wound",
   "metadata": {},
   "source": [
    "### Transform the ACS Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "generous-tennessee",
   "metadata": {},
   "source": [
    "### Transforms for table demographics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "nearby-slope",
   "metadata": {},
   "source": [
    "#### create demo_temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "convertible-brook",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE OR REPLACE TABLE datamart.demo_temp AS\n",
    "SELECT 'US' AS country, null AS category, null AS subcategory1, null AS subcategory2, TRIM(_Label_) AS label, CAST(United_States__Estimate AS NUMERIC) AS estimate, REGEXP_REPLACE(United_States__Margin_of_Error, '[^0-9.]', '') AS error_margin, REGEXP_REPLACE(United_States__Percent, '[^0-9.]', '') AS percent, REGEXP_REPLACE(United_States__Percent_Margin_of_Error, '[^0-9.]', '') AS error_margin_percent\n",
    "FROM us_census_bureau_staging.acs_demographics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "southeast-glossary",
   "metadata": {},
   "source": [
    "#### update blank strings to nulls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "offensive-pavilion",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.demo_temp\n",
    "SET error_margin=null\n",
    "WHERE error_margin=''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "dried-belief",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.demo_temp\n",
    "SET error_margin_percent=null\n",
    "WHERE error_margin_percent=''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "explicit-detector",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.demo_temp\n",
    "SET percent=null\n",
    "WHERE percent=''"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "radio-stock",
   "metadata": {},
   "source": [
    "#### create table demographics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "necessary-might",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE OR REPLACE TABLE datamart.demographics AS\n",
    "SELECT country, CAST(category AS STRING) AS category, CAST(subcategory1 AS STRING) AS subcategory1, CAST(subcategory2 AS STRING) AS subcategory2, label, estimate, CAST(error_margin AS NUMERIC) AS error_margin, CAST(percent AS NUMERIC) AS percent, CAST(error_margin_percent AS NUMERIC) AS error_margin_percent\n",
    "FROM datamart.demo_temp"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "liberal-publicity",
   "metadata": {},
   "source": [
    "#### remove non-data rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "simple-vancouver",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "DELETE FROM datamart.demographics\n",
    "WHERE estimate IS null AND error_margin IS null AND percent IS null AND error_margin IS null"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "iraqi-objective",
   "metadata": {},
   "source": [
    "#### update records with corresponding category and subcategory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "electoral-thing",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.demographics\n",
    "SET category='general'\n",
    "WHERE label IN ('Male', 'Female', 'Sex ratio (males per 100 females)', 'Total population', 'Under 5 years', '5 to 9 years', '10 to 14 years', '15 to 19 years', '20 to 24 years', '25 to 34 years', '35 to 44 years', '45 to 54 years', '55 to 59 years', '60 to 64 years', '65 to 74 years', '75 to 84 years', '85 years and over', 'Median age (years)', 'Under 18 years', '16 years and over', '18 years and over', '21 years and over', '62 years and over', '65 years and over', 'Total housing units', 'Citizen, 18 and over population')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "particular-costs",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.demographics\n",
    "SET subcategory1='sex'\n",
    "WHERE (label='Male' AND estimate=159886919 AND error_margin=5817 AND percent=49.2)\n",
    "OR (label='Female' AND estimate=164810876 AND error_margin=5818 AND percent=50.8)\n",
    "OR (label='Sex ratio (males per 100 females)' AND estimate=97 AND error_margin=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "amended-chick",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.demographics\n",
    "SET subcategory1='age'\n",
    "WHERE label IN ('Under 5 years', '5 to 9 years', '10 to 14 years', '15 to 19 years', '20 to 24 years', '25 to 34 years', '35 to 44 years', '45 to 54 years', '55 to 59 years', '60 to 64 years', '65 to 74 years', '75 to 84 years', '85 years and over', 'Median age (years)', 'Under 18 years', '16 years and over', '18 years and over', '21 years and over', '62 years and over', '65 years and over')\n",
    "OR (label='Male' AND estimate=122360385)\n",
    "OR (label='Female' AND estimate=128908018)\n",
    "OR (label='Male' AND estimate=22518603)\n",
    "OR (label='Female' AND estimate=28265193)\n",
    "OR (label='Sex ratio (males per 100 females)' AND estimate=94.9 AND error_margin=0.1)\n",
    "OR (label='Sex ratio (males per 100 females)' AND estimate=79.7 AND error_margin=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "fallen-confusion",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.demographics\n",
    "SET subcategory2='18 years and over'\n",
    "WHERE (label='18 years and over')\n",
    "OR (label='Male' AND estimate=122360385)\n",
    "OR (label='Female' AND estimate=128908018)\n",
    "OR (label='Sex ratio (males per 100 females)' AND estimate=94.9 AND error_margin=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "norman-window",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.demographics\n",
    "SET subcategory2='65 years and over'\n",
    "WHERE (label='65 years and over')\n",
    "OR (label='Male' AND estimate=22518603)\n",
    "OR (label='Female' AND estimate=28265193)\n",
    "OR (label='Sex ratio (males per 100 females)' AND estimate=79.7 AND error_margin=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "juvenile-identifier",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.demographics\n",
    "SET subcategory1='one race'\n",
    "WHERE label IN ('One race', 'Cherokee tribal grouping', 'Chippewa tribal grouping', 'Navajo tribal grouping', 'Sioux tribal grouping', 'Asian Indian', 'Chinese', 'Filipino', 'Japanese', 'Korean', 'Vietnamese', 'Other Asian', 'Native Hawaiian', 'Guamanian or Chamorro', 'Samoan', 'Other Pacific Islander')\n",
    "OR (label='White' AND estimate=235377662)\n",
    "OR (label='Black or African American' AND estimate=41234642)\n",
    "OR (label='American Indian and Alaska Native' AND estimate=2750143)\n",
    "OR (label='Asian' AND estimate=17924209)\n",
    "OR (label='Native Hawaiian and Other Pacific Islander' AND estimate=599868)\n",
    "OR (label='Some other race' AND estimate=16047369)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "grand-bunny",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.demographics\n",
    "SET subcategory2='American Indian and Alaska Native'\n",
    "WHERE label IN ('Cherokee tribal grouping', 'Chippewa tribal grouping', 'Navajo tribal grouping', 'Sioux tribal grouping')\n",
    "OR (label='American Indian and Alaska Native' AND estimate=2750143)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "great-dependence",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.demographics\n",
    "SET subcategory2='Asian'\n",
    "WHERE label IN ('Asian Indian', 'Chinese', 'Filipino', 'Japanese', 'Korean', 'Vietnamese', 'Other Asian')\n",
    "OR (label='Asian' AND estimate=17924209)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "hungarian-airline",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.demographics\n",
    "SET subcategory2='Native Hawaiian and Other Pacific Islander'\n",
    "WHERE label IN ('Native Hawaiian', 'Guamanian or Chamorro', 'Samoan', 'Other Pacific Islander')\n",
    "OR (label='Native Hawaiian and Other Pacific Islander' AND estimate=599868)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "thick-underwear",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.demographics\n",
    "SET subcategory1='two or more races'\n",
    "WHERE label IN ('White and Black or African American', 'White and American Indian and Alaska Native', 'White and Asian', 'Black or African American and American Indian and Alaska Native')\n",
    "OR (label='Two or more races' AND estimate=10763902)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "jewish-tower",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.demographics\n",
    "SET subcategory1='Race alone or in combination with one or more other races'\n",
    "WHERE (label='White' AND estimate=244597669)\n",
    "OR (label='Black or African American' AND estimate=45612523)\n",
    "OR (label='American Indian and Alaska Native' AND estimate=5643919)\n",
    "OR (label='Asian' AND estimate=21408058)\n",
    "OR (label='Native Hawaiian and Other Pacific Islander' AND estimate=1399393)\n",
    "OR (label='Some other race' AND estimate=17859236)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "graphic-shanghai",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.demographics\n",
    "SET subcategory1='Hispanic or Latino'\n",
    "WHERE label IN ('Hispanic or Latino (of any race)', 'Mexican', 'Puerto Rican', 'Cuban', 'Other Hispanic or Latino')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "separate-profit",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.demographics\n",
    "SET subcategory1='Not Hispanic or Latino'\n",
    "WHERE label IN ('Not Hispanic or Latino', 'White alone', 'Black or African American alone', 'American Indian and Alaska Native alone', 'Asian alone', 'Native Hawaiian and Other Pacific Islander alone', 'Some other race alone','Two races including Some other race', 'Two races excluding Some other race, and Three or more races')\n",
    "OR (label='Two or more races' AND estimate=7941608 AND error_margin=66801 AND percent=2.4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "hawaiian-discharge",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.demographics\n",
    "SET subcategory2='Two or more races'\n",
    "WHERE label IN ('Two races including Some other race', 'Two races excluding Some other race, and Three or more races')\n",
    "OR (label='Two or more races' AND estimate=7941608 AND error_margin=66801 AND percent=2.4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "mechanical-collection",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.demographics\n",
    "SET subcategory1='Citizen, 18 and over population'\n",
    "WHERE (label='Citizen, 18 and over population')\n",
    "OR (label='Male' AND estimate=112003359)\n",
    "OR (label='Female' AND estimate=118956432)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "israeli-asthma",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.demographics\n",
    "SET subcategory2='sex'\n",
    "WHERE (label='Male' AND estimate=112003359)\n",
    "OR (label='Female' AND estimate=118956432)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "appropriate-interstate",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.demographics\n",
    "SET category='race'\n",
    "WHERE subcategory1 IN ('one race', 'two or more races', 'Race alone or in combination with one or more other races', 'Hispanic or Latino', 'Not Hispanic or Latino')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "assigned-finland",
   "metadata": {},
   "source": [
    "#### make fields null in percent column if they're not percentages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "alike-blame",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.demographics\n",
    "SET percent=null\n",
    "WHERE estimate=percent"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "suitable-hawaiian",
   "metadata": {},
   "source": [
    "#### remove duplicate records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "binding-smart",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE OR REPLACE TABLE datamart.demographics AS\n",
    "SELECT DISTINCT *\n",
    "FROM datamart.demographics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "weighted-break",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "DELETE FROM datamart.demographics\n",
    "WHERE (label='18 years and over' AND percent IS null)\n",
    "OR (label='65 years and over' AND percent IS null)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "nervous-ability",
   "metadata": {},
   "source": [
    "#### remove housing record"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "saved-holmes",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "DELETE FROM datamart.demographics\n",
    "WHERE label='Total housing units'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dried-ecuador",
   "metadata": {},
   "source": [
    "#### PK check: if same number then no duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "massive-commitment",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>distinct_PK</th>\n",
       "      <th>demographics_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>81</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   distinct_PK  demographics_count\n",
       "0           81                  81"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT count(*) AS distinct_PK, (SELECT count(*) FROM datamart.demographics) AS demographics_count\n",
    "FROM (\n",
    "    SELECT DISTINCT category, subcategory1, subcategory2, label\n",
    "    FROM datamart.demographics\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "overhead-mozambique",
   "metadata": {},
   "source": [
    "#### FK check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "velvet-cause",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>foreign_key_violations</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   foreign_key_violations\n",
       "0                       0"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT count(*) AS foreign_key_violations\n",
    "FROM datamart.demographics AS d \n",
    "LEFT JOIN datamart.locations AS l ON d.country=l.country\n",
    "WHERE l.id IS null"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "tutorial-transition",
   "metadata": {},
   "source": [
    "### Transforms for table housing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bright-pixel",
   "metadata": {},
   "source": [
    "#### create housing_temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "going-girlfriend",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE OR REPLACE TABLE datamart.housing_temp AS\n",
    "SELECT 'US' AS country, null AS category, null AS subcategory, TRIM(_Label_) AS label, CAST(United_States__Estimate AS NUMERIC) AS estimate, REGEXP_REPLACE(United_States__Margin_of_Error, '[^0-9.]', '') AS error_margin, REGEXP_REPLACE(United_States__Percent, '[^0-9.]', '') AS percent, REGEXP_REPLACE(United_States__Percent_Margin_of_Error, '[^0-9.]', '') AS error_margin_percent\n",
    "FROM us_census_bureau_staging.acs_housing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "structured-matthew",
   "metadata": {},
   "source": [
    "#### update blank strings to nulls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "steady-youth",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing_temp\n",
    "SET error_margin=null\n",
    "WHERE error_margin=''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "portable-tradition",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing_temp\n",
    "SET error_margin_percent=null\n",
    "WHERE error_margin_percent=''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "comparative-identity",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing_temp\n",
    "SET percent=null\n",
    "WHERE percent=''"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "unknown-liabilities",
   "metadata": {},
   "source": [
    "#### create table housing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "mechanical-damages",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE OR REPLACE TABLE datamart.housing AS\n",
    "SELECT country, CAST(category AS STRING) AS category, CAST(subcategory AS STRING) AS subcategory, label, estimate, CAST(error_margin AS NUMERIC) AS error_margin, CAST(percent AS NUMERIC) AS percent, CAST(error_margin_percent AS NUMERIC) AS error_margin_percent\n",
    "FROM datamart.housing_temp"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "intelligent-championship",
   "metadata": {},
   "source": [
    "#### remove non-data rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "bridal-powell",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "DELETE FROM datamart.housing\n",
    "WHERE estimate IS null AND error_margin IS null AND percent IS null AND error_margin IS null"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "unusual-prerequisite",
   "metadata": {},
   "source": [
    "#### update records with corresponding category and subcategory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "written-hudson",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET category='housing units'\n",
    "WHERE label IN ( 'Total housing units', 'Occupied housing units', 'Vacant housing units', 'Homeowner vacancy rate', 'Rental vacancy rate','1-unit, detached', '1-unit, attached','2 units', '3 or 4 units', '5 to 9 units', '10 to 19 units', '20 or more units', 'Mobile home', 'Boat, RV, van, etc.', 'Built 2014 or later', 'Built 2010 to 2013', 'Built 2000 to 2009', 'Built 1990 to 1999', 'Built 1980 to 1989', 'Built 1970 to 1979', 'Built 1960 to 1969', 'Built 1950 to 1959', 'Built 1940 to 1949', 'Built 1939 or earlier', '1 room', '2 rooms', '3 rooms', '4 rooms', '5 rooms', '6 rooms', '7 rooms', '8 rooms', '9 rooms or more', 'Median rooms', 'No bedroom', '1 bedroom', '2 bedrooms', '3 bedrooms', '4 bedrooms', '5 or more bedrooms')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "baking-school",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET subcategory='housing occupancy'\n",
    "WHERE label IN ('Occupied housing units', 'Vacant housing units', 'Homeowner vacancy rate', 'Rental vacancy rate')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cathedral-sugar",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET subcategory='units in structure'\n",
    "WHERE label IN ('1-unit, detached', '1-unit, attached','2 units', '3 or 4 units', '5 to 9 units', '10 to 19 units', '20 or more units', 'Mobile home', 'Boat, RV, van, etc.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "particular-strap",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET subcategory='housing tenure'\n",
    "WHERE label IN ('Owner-occupied', 'Renter-occupied', 'Average household size of owner-occupied unit', 'Average household size of renter-occupied unit')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "mysterious-statistics",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET subcategory='built year'\n",
    "WHERE label IN ('Built 2014 or later', 'Built 2010 to 2013', 'Built 2000 to 2009', 'Built 1990 to 1999', 'Built 1980 to 1989', 'Built 1970 to 1979', 'Built 1960 to 1969', 'Built 1950 to 1959', 'Built 1940 to 1949', 'Built 1939 or earlier')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "mounted-bernard",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET subcategory='rooms'\n",
    "WHERE label IN ('1 room', '2 rooms', '3 rooms', '4 rooms', '5 rooms', '6 rooms', '7 rooms', '8 rooms', '9 rooms or more', 'Median rooms')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "heavy-wellington",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET subcategory='bedrooms'\n",
    "WHERE label IN ('No bedroom', '1 bedroom', '2 bedrooms', '3 bedrooms', '4 bedrooms', '5 or more bedrooms')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "eligible-livestock",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET category='occupied housing units'\n",
    "WHERE label IN ('Owner-occupied', 'Renter-occupied', 'Average household size of owner-occupied unit', 'Average household size of renter-occupied unit', 'Moved in 2017 or later', 'Moved in 2015 to 2016', 'Moved in 2010 to 2014', 'Moved in 2000 to 2009', 'Moved in 1990 to 1999', 'Moved in 1989 and earlier', 'No vehicles available', '1 vehicle available', '2 vehicles available', '3 or more vehicles available', 'Utility gas', 'Bottled, tank, or LP gas', 'Electricity', 'Fuel oil, kerosene, etc.', 'Coal or coke', 'Wood', 'Solar energy', 'Other fuel', 'No fuel used', 'Lacking complete plumbing facilities', 'Lacking complete kitchen facilities', 'No telephone service available', '1.00 or less', '1.01 to 1.50', '1.51 or more')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "nasty-facial",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET subcategory='moved in year'\n",
    "WHERE label IN ('Moved in 2017 or later', 'Moved in 2015 to 2016', 'Moved in 2010 to 2014', 'Moved in 2000 to 2009', 'Moved in 1990 to 1999', 'Moved in 1989 and earlier')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "covered-protest",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET subcategory='vehicles'\n",
    "WHERE label IN ('No vehicles available', '1 vehicle available', '2 vehicles available', '3 or more vehicles available')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "tropical-dress",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET subcategory='house heating fuel'\n",
    "WHERE label IN ('Utility gas', 'Bottled, tank, or LP gas', 'Electricity', 'Fuel oil, kerosene, etc.', 'Coal or coke', 'Wood', 'Solar energy', 'Other fuel', 'No fuel used')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "requested-gambling",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET subcategory='selected characteristics'\n",
    "WHERE label IN ('Lacking complete plumbing facilities', 'Lacking complete kitchen facilities', 'No telephone service available')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "sweet-archives",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET subcategory='occupants per room'\n",
    "WHERE label IN ('1.00 or less', '1.01 to 1.50', '1.51 or more')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "drawn-brook",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET category='owner-occupied units' \n",
    "WHERE label IN ('Owner-occupied units', 'Less than $50,000', '$50,000 to $99,999', '$100,000 to $149,999', '$150,000 to $199,999', '$200,000 to $299,999', '$300,000 to $499,999', '$500,000 to $999,999', '$1,000,000 or more', 'Housing units with a mortgage', 'Housing units without a mortgage')\n",
    "OR (label='Median (dollars)' AND estimate=217500 AND error_margin=180)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "dress-pillow",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET subcategory='value'\n",
    "WHERE label IN ('Less than $50,000', '$50,000 to $99,999', '$100,000 to $149,999', '$150,000 to $199,999', '$200,000 to $299,999', '$300,000 to $499,999', '$500,000 to $999,999', '$1,000,000 or more')\n",
    "OR (label='Median (dollars)' AND estimate=217500 AND error_margin=180)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "governmental-awareness",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET subcategory='mortgage status'\n",
    "WHERE label IN ('Housing units with a mortgage', 'Housing units without a mortgage')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "naked-haven",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET category='housing units with mortgage'\n",
    "WHERE (label='Less than $500' AND estimate=569949 AND error_margin=6363 AND percent=1.2)\n",
    "OR (label='$500 to $999' AND estimate=8242060 AND error_margin=44702 AND percent=17)\n",
    "OR (label='$1,000 to $1,499' AND estimate=13248839 AND error_margin=67426 AND percent=27.4)\n",
    "OR (label='$1,500 to $1,999' AND estimate=10169830 AND error_margin=54768 AND percent=21)\n",
    "OR (label='$2,000 to $2,499' AND estimate=6274095 AND error_margin=36485 AND percent=13)\n",
    "OR (label='$2,500 to $2,999' AND estimate=3841335 AND error_margin=23746 AND percent=7.9)\n",
    "OR (label='$3,000 or more' AND estimate=6070519 AND error_margin=31722 AND percent=12.5)\n",
    "OR (label='Median (dollars)' AND estimate=1595 AND error_margin=2)\n",
    "OR (label='Less than 20.0 percent')\n",
    "OR (label='20.0 to 24.9 percent' AND estimate=7584745 AND error_margin=48798 AND percent=15.7)\n",
    "OR (label='25.0 to 29.9 percent' AND estimate=5072173 AND error_margin=29346 AND percent=10.5)\n",
    "OR (label='30.0 to 34.9 percent' AND estimate=3330974 AND error_margin=18266 AND percent=6.9)\n",
    "OR (label='35.0 percent or more' AND estimate=10069038 AND error_margin=23122 AND percent=20.9)\n",
    "OR (label='Not computed' AND estimate=233653 AND error_margin=3535)\n",
    "OR (label='Housing units with a mortgage')\n",
    "OR (label='Housing units with a mortgage (excluding units where SMOCAPI cannot be computed)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "stuck-equation",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET subcategory='selected monthly owner costs'\n",
    "WHERE (label='Less than $500' AND estimate=569949 AND error_margin=6363 AND percent=1.2)\n",
    "OR (label='$500 to $999' AND estimate=8242060 AND error_margin=44702 AND percent=17)\n",
    "OR (label='$1,000 to $1,499' AND estimate=13248839 AND error_margin=67426 AND percent=27.4)\n",
    "OR (label='$1,500 to $1,999' AND estimate=10169830 AND error_margin=54768 AND percent=21)\n",
    "OR (label='$2,000 to $2,499' AND estimate=6274095 AND error_margin=36485 AND percent=13)\n",
    "OR (label='$2,500 to $2,999' AND estimate=3841335 AND error_margin=23746 AND percent=7.9)\n",
    "OR (label='$3,000 or more' AND estimate=6070519 AND error_margin=31722 AND percent=12.5)\n",
    "OR (label='Median (dollars)' AND estimate=1595 AND error_margin=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "metric-joshua",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET subcategory='selected monthly owner costs as a percentage of household income'\n",
    "WHERE (label='Less than 20.0 percent')\n",
    "OR (label='20.0 to 24.9 percent' AND estimate=7584745 AND error_margin=48798 AND percent=15.7)\n",
    "OR (label='25.0 to 29.9 percent' AND estimate=5072173 AND error_margin=29346 AND percent=10.5)\n",
    "OR (label='30.0 to 34.9 percent' AND estimate=3330974 AND error_margin=18266 AND percent=6.9)\n",
    "OR (label='35.0 percent or more' AND estimate=10069038 AND error_margin=23122 AND percent=20.9)\n",
    "OR (label='Not computed' AND estimate=233653 AND error_margin=3535)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "judicial-middle",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET category='housing units without mortgage'\n",
    "WHERE label IN ('Less than $250', '$250 to $399', '$400 to $599', '$600 to $799', '$800 to $999', '$1,000 or more', 'Less than 10.0 percent', '10.0 to 14.9 percent')\n",
    "OR (label='Median (dollars)' AND estimate=500 AND error_margin=1)\n",
    "OR (label='15.0 to 19.9 percent' AND estimate=3154745 AND error_margin=16364 AND percent=11.1)\n",
    "OR (label='20.0 to 24.9 percent' AND estimate=1916515 AND error_margin=11272 AND percent=6.7)\n",
    "OR (label='25.0 to 29.9 percent' AND estimate=1235715 AND error_margin=7526 AND percent=4.3)\n",
    "OR (label='30.0 to 34.9 percent' AND estimate=836924 AND error_margin=6646 AND percent=2.9)\n",
    "OR (label='35.0 percent or more' AND estimate=3010014 AND error_margin=15340 AND percent=10.6)\n",
    "OR (label='Not computed' AND estimate=394725 AND error_margin=4268)\n",
    "OR (label='Housing units without a mortgage')\n",
    "OR (label='Housing unit without a mortgage (excluding units where SMOCAPI cannot be computed)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "threatened-fellow",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET subcategory='selected monthly owner costs'\n",
    "WHERE label IN ('Less than $250', '$250 to $399', '$400 to $599', '$600 to $799', '$800 to $999', '$1,000 or more')\n",
    "OR (label='Median (dollars)' AND estimate=500 AND error_margin=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "published-destination",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET subcategory='selected monthly owner costs as a percentage of household income'\n",
    "WHERE label IN ('Less than 10.0 percent', '10.0 to 14.9 percent')\n",
    "OR (label='15.0 to 19.9 percent' AND estimate=3154745 AND error_margin=16364 AND percent=11.1)\n",
    "OR (label='20.0 to 24.9 percent' AND estimate=1916515 AND error_margin=11272 AND percent=6.7)\n",
    "OR (label='25.0 to 29.9 percent' AND estimate=1235715 AND error_margin=7526 AND percent=4.3)\n",
    "OR (label='30.0 to 34.9 percent' AND estimate=836924 AND error_margin=6646 AND percent=2.9)\n",
    "OR (label='35.0 percent or more' AND estimate=3010014 AND error_margin=15340 AND percent=10.6)\n",
    "OR (label='Not computed' AND estimate=394725 AND error_margin=4268)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "regular-instruction",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET category='occupied units paying rent'\n",
    "WHERE (label='Less than $500' AND estimate=3865037 AND error_margin=13581 AND percent=9.4)\n",
    "OR (label='$500 to $999' AND estimate=14956085 AND error_margin=47591 AND percent=36.2)\n",
    "OR (label='$1,000 to $1,499' AND estimate=12403502 AND error_margin=50982 AND percent=30)\n",
    "OR (label='$1,500 to $1,999' AND estimate=5796670 AND error_margin=32779 AND percent=14)\n",
    "OR (label='$2,000 to $2,499' AND estimate=2324841 AND error_margin=14722 AND percent=5.6)\n",
    "OR (label='$2,500 to $2,999' AND estimate=987766 AND error_margin=8283 AND percent=2.4)\n",
    "OR (label='$3,000 or more' AND estimate=977971 AND error_margin=7308 AND percent=2.4)\n",
    "OR (label='Median (dollars)' AND estimate=1062 AND error_margin=1)\n",
    "OR (label='No rent paid' AND estimate=2169795 AND error_margin=10848)\n",
    "OR (label='Less than 15.0 percent')\n",
    "OR (label='15.0 to 19.9 percent' AND estimate=5195435 AND error_margin=23159 AND percent=12.9)\n",
    "OR (label='20.0 to 24.9 percent' AND estimate=5190296 AND error_margin=21539 AND percent=12.9)\n",
    "OR (label='25.0 to 29.9 percent' AND estimate=4681621 AND error_margin=19362 AND percent=11.6)\n",
    "OR (label='30.0 to 34.9 percent' AND estimate=3674673 AND error_margin=17612 AND percent=9.1)\n",
    "OR (label='35.0 percent or more' AND estimate=16328272 AND error_margin=67971 AND percent=40.5)\n",
    "OR (label='Not computed' AND estimate=3115329 AND error_margin=16031)\n",
    "OR (label='Occupied units paying rent')\n",
    "OR (label='Occupied units paying rent (excluding units where GRAPI cannot be computed)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "swiss-isolation",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET subcategory='gross rent'\n",
    "WHERE (label='Less than $500' AND estimate=3865037 AND error_margin=13581 AND percent=9.4)\n",
    "OR (label='$500 to $999' AND estimate=14956085 AND error_margin=47591 AND percent=36.2)\n",
    "OR (label='$1,000 to $1,499' AND estimate=12403502 AND error_margin=50982 AND percent=30)\n",
    "OR (label='$1,500 to $1,999' AND estimate=5796670 AND error_margin=32779 AND percent=14)\n",
    "OR (label='$2,000 to $2,499' AND estimate=2324841 AND error_margin=14722 AND percent=5.6)\n",
    "OR (label='$2,500 to $2,999' AND estimate=987766 AND error_margin=8283 AND percent=2.4)\n",
    "OR (label='$3,000 or more' AND estimate=977971 AND error_margin=7308 AND percent=2.4)\n",
    "OR (label='Median (dollars)' AND estimate=1062 AND error_margin=1)\n",
    "OR (label='No rent paid' AND estimate=2169795 AND error_margin=10848)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "enclosed-jewel",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET subcategory='gross rent as a percentage of household income'\n",
    "WHERE (label='Less than 15.0 percent')\n",
    "OR (label='15.0 to 19.9 percent' AND estimate=5195435 AND error_margin=23159 AND percent=12.9)\n",
    "OR (label='20.0 to 24.9 percent' AND estimate=5190296 AND error_margin=21539 AND percent=12.9)\n",
    "OR (label='25.0 to 29.9 percent' AND estimate=4681621 AND error_margin=19362 AND percent=11.6)\n",
    "OR (label='30.0 to 34.9 percent' AND estimate=3674673 AND error_margin=17612 AND percent=9.1)\n",
    "OR (label='35.0 percent or more' AND estimate=16328272 AND error_margin=67971 AND percent=40.5)\n",
    "OR (label='Not computed' AND estimate=3115329 AND error_margin=16031)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fresh-hindu",
   "metadata": {},
   "source": [
    "#### make fields null in percent column if they're not percentages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "proud-contrary",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.housing\n",
    "SET percent=null\n",
    "WHERE estimate=percent"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "extended-trade",
   "metadata": {},
   "source": [
    "#### remove duplicate records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "vanilla-wrestling",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE OR REPLACE TABLE datamart.housing AS\n",
    "SELECT DISTINCT *\n",
    "FROM datamart.housing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "numeric-chicken",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "DELETE FROM datamart.housing\n",
    "WHERE (label='Occupied housing units' AND percent IS null)\n",
    "OR (label='Housing units with a mortgage' AND percent IS null)\n",
    "OR (label='Housing units without a mortgage' AND percent IS null)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "thousand-length",
   "metadata": {},
   "source": [
    "#### PK check: if same number then no duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "seventh-secretariat",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>distinct_PK</th>\n",
       "      <th>housing_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>130</td>\n",
       "      <td>130</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   distinct_PK  housing_count\n",
       "0          130            130"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT count(*) AS distinct_PK, (SELECT count(*) FROM datamart.housing) AS housing_count\n",
    "FROM (\n",
    "    SELECT DISTINCT category, subcategory, label\n",
    "    FROM datamart.housing\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "chicken-surgery",
   "metadata": {},
   "source": [
    "#### FK check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "cosmetic-customer",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>foreign_key_violations</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   foreign_key_violations\n",
       "0                       0"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT count(*) AS foreign_key_violations\n",
    "FROM datamart.housing AS h\n",
    "LEFT JOIN datamart.locations AS l ON h.country=l.country\n",
    "WHERE l.id IS null"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "present-traffic",
   "metadata": {},
   "source": [
    "### Transforms for table economics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "advisory-vessel",
   "metadata": {},
   "source": [
    "#### Create economics temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "anticipated-avatar",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE OR REPLACE TABLE datamart.economics_temp AS\n",
    "SELECT 'US' AS country, null AS category, null AS subcategory, TRIM (Label) AS label, CAST(United_States__Estimate AS NUMERIC) AS estimate, REGEXP_REPLACE(United_States__Margin_of_Error, '[^0-9.]', '') AS error_margin, REGEXP_REPLACE(United_States__Percent, '[^0-9.]', '') AS percent, REGEXP_REPLACE(United_States__Percent_Margin_of_Error, '[^0-9.]', '') AS error_margin_percent\n",
    "FROM us_census_bureau_staging.acs_economics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "parliamentary-penetration",
   "metadata": {},
   "source": [
    "#### Update blank strings to nulls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "laden-phoenix",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.economics_temp\n",
    "SET error_margin=null\n",
    "WHERE error_margin=''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "nervous-title",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.economics_temp\n",
    "SET error_margin_percent=null\n",
    "WHERE error_margin_percent=''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "urban-groove",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.economics_temp\n",
    "SET percent=null\n",
    "WHERE percent=''"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "vocal-partition",
   "metadata": {},
   "source": [
    "#### Create Table Economics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "periodic-trace",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE OR REPLACE TABLE datamart.economics AS\n",
    "SELECT country, CAST(category AS STRING) AS category, CAST(subcategory AS STRING) AS subcategory, label, estimate, CAST(error_margin AS NUMERIC) AS error_margin, CAST(percent AS NUMERIC) AS percent, CAST(error_margin_percent AS NUMERIC) AS error_margin_percent\n",
    "FROM datamart.economics_temp"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "imperial-destination",
   "metadata": {},
   "source": [
    "#### Remove non-data rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "blond-hartford",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "DELETE FROM datamart.economics\n",
    "WHERE estimate IS null AND error_margin IS null AND percent IS null AND error_margin IS null"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "mysterious-sleeping",
   "metadata": {},
   "source": [
    "#### Update records with corresponding category and subcategory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "genuine-swaziland",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "UPDATE datamart.economics\n",
    "SET category = 'employment'\n",
    "WHERE label IN ('Civilian employed population 16 years and over', 'In labor force', 'Civilian labor force', 'Armed Forces', 'Not in labor force', 'Civilian labor force', 'Unemployment Rate', 'All parents in family in labor force', 'Car, truck, or van -- drove alone', 'Car, truck, or van -- carpooled', 'Public transportation (excluding taxicab)', 'Walked', 'Other means', 'Worked from home', 'Mean travel time to work (minutes)', 'Agriculture, forestry, fishing and hunting, and mining', 'Construction', 'Manufacturing', 'Wholesale trade', 'Retail trade', 'Transportation and warehousing, and utilities', 'Information', 'Finance and insurance, and real estate and rental and leasing', 'Professional, scientific, and management, and administrative and waste management services', 'Educational services, and health care and social assistance', 'Arts, entertainment, and recreation, and accommodation and food services', 'Other services, except public administration', 'Public administration', 'Private wage and salary workers', 'Government workers', 'Self-employed in own not incorporated business workers', 'Unpaid family workers', 'Less than $10,000', '$10,000 to $14,999', '$15,000 to $24,999', '$25,000 to $34,999', '$35,000 to $49,999' , '$50,000 to $74,999', '$75,000 to $99,999', '$100,000 to $149,999', '$150,000 to $199,999', '$200,000 or more', 'Median household income (dollars)', 'Mean household income (dollars)', 'With earnings', 'Mean earnings (dollars)', 'With Social Security', 'Mean Social Security income (dollars)', 'With retirement income','Mean retirement income (dollars)', 'With Supplemental Security income (dollars)', 'Mean Supplemental Security Income (dollars)', 'With cash public assistance income', 'Mean cash public assistance income (dollars)', 'With Food Stamp/SNAP benefits in the past 12 months', 'Nonfamily households', 'Median nonfamily income (dollars)', 'Mean nonfamily income (dollars)', 'Median earnings for workers (dollars)', 'Median earnings for male full-time, year-round workers (dollars)', 'Median earnings for female full-time, year-round workers (dollars)', 'Employed:', \"Unemployed:\", \"Workers 16 years and over\", 'Mean family income (dollars)', 'Median family income (dollars)', 'Per capita income (dollars)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "basic-fellowship",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "UPDATE datamart.economics\n",
    "SET category = 'general'\n",
    "WHERE label IN ('All people','Families', 'All families', '18 to 64 years', 'Married couple families', 'People in families', '18 years and over', '65 years and over' ,'Population 16 years and over', 'Total households', 'Females 16 years and over','Own children of the householder under 6 years', 'Own children of the householder 6 to 17 years', 'Management, business, science, and arts occupations', 'Service occupations', 'Sales and office occupations', 'Natural resources, construction, and maintenance occupations', 'Production, transportation, and material moving occupations', 'With related children of the householder under 5 years only', 'Related children of the householder under 5 years', 'With related children of the householder under 18 years', 'Related children of the householder 5 to 17 years', 'Under 18 years', 'Families with female householder, no spouse present', 'Unrelated individuals 15 years and over','Related children of the householder under 18 years', \"With Supplemental Security Income\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "acute-black",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "UPDATE datamart.economics\n",
    "SET subcategory = 'employment status'\n",
    "WHERE (label = 'Population 16 years and over')\n",
    "OR (label = 'Workers 16 years and over')\n",
    "OR (label ='Total households')\n",
    "OR (label ='Civilian employed population 16 years and over')\n",
    "OR (label = 'In labor force:')\n",
    "OR (label = 'Armed Forces')\n",
    "OR (label = 'Not in labor force:')\n",
    "OR (label = 'Civilian labor force' AND estimate = 163555585 AND percent = .63)\n",
    "OR (label = 'Unemployment Rate')\n",
    "OR (label = 'Unemployed:')\n",
    "OR (label = 'Employed:')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "handled-history",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "UPDATE datamart.economics\n",
    "SET subcategory = 'females 16 years and over'\n",
    "WHERE (label ='Females 16 years and over')\n",
    "OR (label = 'Civilian labor force' AND estimate = 77596212)\n",
    "OR (label = 'Employed' AND estimate = 73536561)\n",
    "OR (label = 'In labor force' AND estimate = 77750312)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "listed-program",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "UPDATE datamart.economics\n",
    "SET subcategory = 'children of the householder'\n",
    "WHERE label in ('Own children of the householder under 6 years')\n",
    "OR (label = 'Own children of the householder 6 to 17 years')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "compatible-greenhouse",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.economics\n",
    "SET subcategory = 'commuting to work'\n",
    "WHERE label in ('Car, truck, or van -- drove alone', 'Car, truck, or van -- carpooled', 'Public transportation (excluding taxicab)', 'Walked', 'Other means', 'Worked from home', 'Mean travel time to work (minutes)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "satisfied-hygiene",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.economics \n",
    "SET subcategory = 'occupation'\n",
    "WHERE label in ('Management, business, science, and arts occupations', 'Service occupations', 'Sales and office occupations', 'Natural resources, construction, and maintenance occupations', 'Production, transportation, and material moving occupations')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "selective-orbit",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.economics\n",
    "SET subcategory = 'industry'\n",
    "WHERE label in ('Agriculture, forestry, fishing and hunting, and mining', 'Construction', 'Manufacturing', 'Wholesale trade', 'Retail trade', 'Transportation and warehousing, and utilities', 'Information', 'Finance and insurance, and real estate and rental and leasing', 'Professional, scientific, and management, and administrative and waste management services', 'Educational services, and health care and social assistance', 'Arts, entertainment, and recreation, and accommodation and food services', 'Other services, except public administration', 'Public administration')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "common-syndrome",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.economics\n",
    "SET subcategory = 'class of worker'\n",
    "WHERE label in ('Private wage and salary workers', 'Government workers', 'Self-employed in own not incorporated business workers', 'Unpaid family workers')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "convertible-river",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.economics\n",
    "SET subcategory = 'income'\n",
    "WHERE (label = 'Less than $10,000' AND estimate = 7302871)\n",
    "OR (label = '$10,000 to $14,999' AND estimate = 5189583)\n",
    "OR (label = '$15,000 to $24,999' AND estimate = 10760144)\n",
    "OR (label = '$25,000 to $34,999' AND estimate = 10792134)\n",
    "OR (label = '$35,000 to $49,999' AND estimate = 14822045)\n",
    "OR (label = '$50,000 to $74,999' AND estimate = 20789890)\n",
    "OR (label = '$75,000 to $99,999' AND estimate = 15374617)\n",
    "OR (label = '$100,000 to $149,999' AND estimate = 18286811)\n",
    "OR (label = '$150,000 to $199,999' AND estimate = 8173563)\n",
    "OR (label = '$200,000 or more' AND estimate = 9264390)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "current-polish",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.economics\n",
    "SET subcategory = 'benefits'\n",
    "WHERE label in ('Median household income (dollars)', 'Mean household income (dollars)', 'With earnings', 'Mean earnings (dollars)', 'With Social Security', 'Mean Social Security income (dollars)', 'With retirement income','Mean retirement income (dollars)', 'With Supplemental Security Income', 'Mean Supplemental Security Income (dollars)', 'With cash public assistance income', 'Mean cash public assistance income (dollars)', 'With Food Stamp/SNAP benefits in the past 12 months')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "lined-celebration",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.economics\n",
    "SET subcategory = 'families income'\n",
    "WHERE (label ='Less than $10,000' AND estimate = 2905334)\n",
    "OR (label = '$10,000 to $14,999' AND estimate = 1835765)\n",
    "OR (label = '$15,000 to $24,999' AND estimate = 4904304)\n",
    "OR (label = '$25,000 to $34,999' AND estimate = 5896133)\n",
    "OR (label = '$35,000 to $49,999' AND estimate = 8950976)\n",
    "OR (label = '$50,000 to $74,999' AND estimate = 13867897)\n",
    "OR (label = '$75,000 to $99,999' AND estimate = 11374396)\n",
    "OR (label = '$100,000 to $149,999' AND estimate = 14620038)\n",
    "OR (label = '$150,000 to $199,999' AND estimate = 6852461)\n",
    "OR (label = '$200,000 or more' AND estimate = 7906727)\n",
    "OR (label = 'Median family income (dollars)')\n",
    "OR (label = 'Mean family income (dollars)')\n",
    "OR (label = 'Per capita income (dollars)')\n",
    "OR (label = 'Families')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "informal-hindu",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.economics\n",
    "SET subcategory = 'nonfamily households'\n",
    "WHERE label in ('Nonfamily households', 'Median nonfamily income (dollars)', 'Mean nonfamily income (dollars)', 'Median earnings for workers (dollars)', 'Median earnings for male full-time, year-round workers (dollars)', 'Median earnings for female full-time, year-round workers (dollars)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "legal-firmware",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.economics\n",
    "SET category = 'Health Insurance Coverage'\n",
    "WHERE (label = 'Civilian noninstitutionalized population', 'With health insurance coverage', 'With private health insurance', 'With public coverage', 'No health insurance coverage', 'Civilian noninstitutionalized population under 19 years', 'Civilian noninstitutionalized population 19 to 64 years', \"Employed:\", 'Unemployed:', 'In labor force:', 'Not in labor force:')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "worthy-castle",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "UPDATE datamart.economics \n",
    "SET subcategory = 'Civilian noninstutionalized population'\n",
    "WHERE (label= 'Civilian noninstitutionalized population')\n",
    "OR (label = 'With health insurance coverage' AND estimate = 291458259)\n",
    "OR (label = 'With private health insurance' AND estimate = 217186300)\n",
    "OR (label = 'With public coverage' AND estimate = 112236495)\n",
    "OR (label = 'No health insurance coverage' AND estimate = 29249613)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "northern-chicago",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "UPDATE datamart.economics \n",
    "SET subcategory = 'Civilian noninstutionalized population under 19 years'\n",
    "WHERE (label = 'No health insurance coverage' AND estimate = 3945906)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "patent-supplement",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "UPDATE datamart.economics \n",
    "SET subcategory = 'Civilian noninstutionalized population 19 to 64 years'\n",
    "WHERE (label= 'In labor force:')\n",
    "OR (label = 'Employed:')\n",
    "OR (label = 'Unemployed:')\n",
    "OR (label = 'Not in labor force:')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "amber-sensitivity",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "DELETE FROM datamart.economics\n",
    "WHERE (label='With health insurance coverage' AND estimate != 291458259)\n",
    "OR (label='With private health insurance' AND estimate != 217186300)\n",
    "OR (label='With public coverage' AND estimate != 112236495)\n",
    "OR (label='No health insurance coverage' AND (estimate != 29249613 AND estimate != 3945906))\n",
    "OR (label = 'All parents in family in labor force')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "consecutive-collar",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "UPDATE datamart.economics\n",
    "SET subcategory = 'percent below poverty level'\n",
    "WHERE label in ('All families', 'Married couple families', 'Families with female householder, no spouse present', 'All people', 'Under 18 years', '18 years and over', '18 to 64 years', '65 years and over', 'People in families', 'Unrelated individuals 15 years and over')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "published-refund",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "DELETE FROM datamart.economics \n",
    "WHERE (label ='With related children of the householder under 18 years')\n",
    "OR (label = 'With related children of the householder under 5 years only')\n",
    "OR (label = 'Related children of the householder 5 to 17 years')\n",
    "OR (label = 'Civilian labor force' AND percent = 163555585)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "editorial-baseball",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "%%bigquery\n",
    "DELETE FROM datamart.economics\n",
    "WHERE (subcategory='employment status' AND label='Civilian labor force' AND percent IS null)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "subjective-accident",
   "metadata": {},
   "source": [
    "#### Make fields null in percent column if they are not percentages\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "hidden-wayne",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.economics\n",
    "SET percent=null\n",
    "WHERE estimate=percent"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "tough-creature",
   "metadata": {},
   "source": [
    "#### Remove Duplicate Records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "transsexual-invitation",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE OR REPLACE TABLE datamart.economics AS\n",
    "SELECT DISTINCT *\n",
    "FROM datamart.economics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "statistical-witness",
   "metadata": {},
   "source": [
    "#### Primary Key Check: If they are the same number then there are no duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "nominated-pendant",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>distinct_PK</th>\n",
       "      <th>economics_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>112</td>\n",
       "      <td>112</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   distinct_PK  economics_count\n",
       "0          112              112"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT count(*) AS distinct_PK, (SELECT count(*) FROM datamart.economics) AS economics_count\n",
    "FROM (\n",
    "    SELECT DISTINCT category, subcategory, label\n",
    "    FROM datamart.economics\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "liable-phase",
   "metadata": {},
   "source": [
    "#### Foreign Key check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "indoor-royalty",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>foreign_key_violations</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   foreign_key_violations\n",
       "0                       0"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT count(*) AS foreign_key_violations\n",
    "FROM datamart.economics AS h\n",
    "LEFT JOIN datamart.locations AS l ON h.country=l.country\n",
    "WHERE l.id IS null"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "exceptional-despite",
   "metadata": {},
   "source": [
    "### Transforms for table social"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "revised-lewis",
   "metadata": {},
   "source": [
    "#### Create social temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "opened-property",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE OR REPLACE TABLE datamart.social_temp AS\n",
    "SELECT 'US' AS country, null AS category, null AS subcategory, TRIM(_Label_) AS label, CAST(United_States__Estimate AS NUMERIC) AS estimate, REGEXP_REPLACE(United_States__Margin_of_Error, '[^0-9.]', '') AS error_margin, REGEXP_REPLACE(United_States__Percent, '[^0-9.]', '') AS percent, REGEXP_REPLACE(United_States__Percent_Margin_of_Error, '[^0-9.]', '') AS error_margin_percent\n",
    "FROM us_census_bureau_staging.acs_social"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "elegant-biology",
   "metadata": {},
   "source": [
    "#### Update Blank Strings to nulls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "advised-asbestos",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.social_temp\n",
    "SET error_margin=null\n",
    "WHERE error_margin=''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "smaller-vocabulary",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.social_temp\n",
    "SET error_margin_percent=null\n",
    "WHERE error_margin_percent=''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "found-brush",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.social_temp\n",
    "SET percent=null\n",
    "WHERE percent=''"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "empirical-breakfast",
   "metadata": {},
   "source": [
    "#### Create Table Social"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "gothic-cotton",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE OR REPLACE TABLE datamart.social AS\n",
    "SELECT country, CAST(category AS STRING) AS category, CAST(subcategory AS STRING) AS subcategory, label, estimate, CAST(error_margin AS NUMERIC) AS error_margin, CAST(percent AS NUMERIC) AS percent, CAST(error_margin_percent AS NUMERIC) AS error_margin_percent \n",
    "FROM datamart.social_temp"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "judicial-departure",
   "metadata": {},
   "source": [
    "#### Remove non data rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "precise-jacket",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "DELETE FROM datamart.social\n",
    "WHERE estimate IS null AND error_margin IS null AND percent IS null AND error_margin IS null"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "architectural-monaco",
   "metadata": {},
   "source": [
    "##### Update records with corresponding category and sub category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "laden-syndrome",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "UPDATE datamart.social\n",
    "SET category = 'relationships'\n",
    "WHERE label IN ('Population in households', 'Householder', 'Spouse', 'Unmarried partner', 'Child', 'Other relatives', 'Other nonrelatives','Never married', 'Males 15 years and over', 'Now married, except separated', 'Separated', 'Widowed','Divorced', 'Females 15 years and over', 'Number of women 15 to 50 years old who had a birth in the past 12 months', 'Unmarried women (widowed, divorced, and never married)', 'Per 1,000 unmarried women', 'Per 1,000 women 15 to 50 years old', 'Per 1,000 women 15 to 19 years old', 'Per 1,000 women 20 to 34 years old', 'Per 1,000 women 35 to 50 years old', 'Number of grandparents living with own grandchildren under 18 years', 'Grandparents responsible for grandchildren',  'Less than 1 year', '1 or 2 years', '3 or 4 years', '5 or more years', 'Number of grandparents responsible for own grandchildren under 18 years')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "physical-governor",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.social\n",
    "SET category = 'general'\n",
    "WHERE label IN ('Population 3 years and over enrolled in school', 'Nursery school,preschool', 'Kindergarten', 'Elementary school (grades 1-8)', 'High school (grades 9-12)', 'College or graduate school', 'Population 25 years and over', 'Less than 9th grade', '9th to 12th grade, no diploma', 'High school graduate (includes equivalency)', 'Some college, no degree', \"Associate's degree\", \"Bachelor's degree\", 'Graduate or professional degree', 'High school graduate or higher', \"Bachelor's degree or higher\", 'Civilian population 18 years and over', 'Civilian veterans', 'Total Civilian Noninstitutionalized Population', 'Under 18 years', '18 to 64 years', '65 years and over', 'With a computer', 'With a broadband Internet subscription')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "relative-grain",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "UPDATE datamart.social\n",
    "SET category = 'residence'\n",
    "WHERE label IN ('Total households', 'Married-couple family', 'Population 1 year and over', 'Cohabiting couple household', 'Male householder, no spouse/partner present', 'Female householder, no spouse/partner present','Households with one or more people under 18 years','Households with one or more people 65 years and over', 'Average household size','Average family size', 'Same house', 'Different house in the U.S.', 'Same county', 'Different county', 'Same state', 'Different state', 'Abroad', 'Total population', 'Born in the  United States', \"State of residence\",'residence 1 year ago', 'Born in Puerto Rico, U.S. Island areas, or born abroad to American parent(s)', 'Foreign born')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "suburban-twelve",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "UPDATE datamart.social\n",
    "SET category = 'culture and ancestry'\n",
    "WHERE label IN ('Foreign-born population', 'Population born outside the United States', 'Population 5 years and over', 'Naturalized U.S. citizen', 'Not a U.S. citizen', 'Peopleborn outside the United States', 'Native', 'Foreign born', 'Foreign-born population, excluding population born at sea', 'Europe', 'Asia', 'Africa', 'Oceania', 'Latin America', 'Northern America', 'English only', 'Language other than English', \"Spanish\", 'Other Indo-European languages', 'Asian and Pacific Islander languages', 'Other languages', 'Total population', 'American', 'Arab', 'Czech', 'Danish', 'Dutch', 'English', 'French (except Basque)', 'French Canadian', 'German', 'Greek', 'Hungarian', 'Irish', 'Italian', 'Lithuanian', 'Norwegian', 'Polish', 'Portuguese', 'Russian', 'Scotch-Irish', 'Scottish', 'Slovak', 'Subsaharan African', 'Swedish', 'Swiss', 'Ukrainian', 'Welsh', 'West Indian (excluding Hispanic origin groups)' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "palestinian-commitment",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "UPDATE datamart.social\n",
    "SET subcategory = 'household types'\n",
    "WHERE (label = 'Total households')\n",
    "OR (label = 'Married-couple family')\n",
    "OR (label = 'Cohabiting couple household')\n",
    "OR (label = 'Male householder, no spouse/partner present')\n",
    "OR (label = 'Female householder, no spouse/partner present')\n",
    "OR (label = 'Households with one or more people under 18 years')\n",
    "OR (label = 'Households with one or more people 65 years and over')\n",
    "OR (label = 'Average household size')\n",
    "OR (label = 'Average family size')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "narrow-advancement",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "DELETE from datamart.social \n",
    "WHERE (label = 'With own children of the householder under 18 years')\n",
    "OR (label = 'Householder living alone')\n",
    "OR (label = 'Entered 2010 or later')\n",
    "OR (label = 'Entered before 2010')\n",
    "OR (label = '65 years and over' AND estimate = 4227728)\n",
    "OR (label = '65 years and over' AND estimate = 9032038)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "authentic-jamaica",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "UPDATE datamart.social \n",
    "SET subcategory = 'relationships in household'\n",
    "WHERE label in ('Population in households', 'Householder', 'Spouse', 'Unmarried partner', 'Child', 'Other relatives', 'Other nonrelatives')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "meaningful-scotland",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "UPDATE datamart.social\n",
    "SET subcategory = 'males marital status'\n",
    "WHERE (label = 'Never married' and estimate = 47039874)\n",
    "OR (label = 'Males 15 years and over')\n",
    "OR (label = 'Now married, except separated' and estimate = 64014364)\n",
    "OR (label = 'Separated' and estimate = 2094580)\n",
    "OR (label = 'Widowed' and estimate = 3396202)\n",
    "OR (label = 'Divorced' and estimate = 12244434)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "decimal-portsmouth",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "UPDATE datamart.social\n",
    "SET subcategory = 'females marital status'\n",
    "WHERE (label = 'Never married' and estimate = 41019187)\n",
    "OR (label = 'Females 15 years and over')\n",
    "OR (label = 'Now married, except separated' and estimate = 62809181)\n",
    "OR (label = 'Separated' and estimate = 2950832)\n",
    "OR (label = 'Widowed' and estimate = 11878769)\n",
    "OR (label = 'Divorced' and estimate = 16397947)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "representative-disability",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.social \n",
    "SET subcategory = 'fertility'\n",
    "WHERE label in ('Number of women 15 to 50 years old who had a birth in the past 12 months', 'Unmarried women (widowed, divorced, and never married)', 'Per 1,000 unmarried women', 'Per 1,000 women 15 to 50 years old', 'Per 1,000 women 15 to 19 years old', 'Per 1,000 women 20 to 34 years old', 'Per 1,000 women 35 to 50 years old')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "average-scene",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "UPDATE datamart.social\n",
    "SET subcategory = 'grandparents'\n",
    "WHERE label in ('Number of grandparents living with own grandchildren under 18 years', 'Grandparents responsible for grandchildren',  'Less than 1 year', '1 or 2 years', '3 or 4 years', '5 or more years', 'Number of grandparents responsible for own grandchildren under 18 years')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "stretch-resident",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "DELETE from datamart.social \n",
    "WHERE label in ('Who are female', 'Who are married', 'With a disability', 'Speak English less than \"very well\"')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "athletic-college",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "UPDATE datamart.social \n",
    "SET subcategory = 'School Enrollment'\n",
    "WHERE label in ('Population 3 years and over enrolled in school', 'Nursery school, preschool', 'Kindergarten', 'Elementary school (grades 1-8)', 'High school (grades 9-12)', 'College or graduate school')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "confident-combine",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.social\n",
    "SET subcategory = 'Educational Attainment'\n",
    "WHERE label in ('Population 25 years and over', 'Less than 9th grade', '9th to 12th grade, no diploma', 'High school graduate (includes equivalency)', 'Some college, no degree', \"Associate's degree\", \"Bachelor's degree\", 'Graduate or professional degree', 'High school graduate or higher', \"Bachelor's degree or higher\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "liked-summit",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "UPDATE datamart.social\n",
    "SET subcategory = 'Veteran Status'\n",
    "WHERE label in ('Civilian population 18 years and over', 'Civilian veterans', 'Total Civilian Noninstitutionalized Population', 'Under 18 years', '18 to 64 years', '65 years and over')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "expressed-former",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "UPDATE datamart.social\n",
    "SET subcategory = 'residence 1 year ago'\n",
    "WHERE (label = 'Population 1 year and over')\n",
    "OR (label = 'Same house')\n",
    "OR (label = 'Different house in the U.S.')\n",
    "OR (label = 'Same county')\n",
    "OR (label ='Different county')\n",
    "OR (label = 'Same state')\n",
    "OR (label = 'Different state' AND estimate = 7475038)\n",
    "OR (label = 'Abroad')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "friendly-azerbaijan",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.social\n",
    "SET subcategory = 'U.S. citizenship status'\n",
    "WHERE (label = 'Total population')\n",
    "OR (label = 'Native' AND estimate = 280685925) \n",
    "OR (label ='Born in United States')\n",
    "OR (label = 'State of residence')\n",
    "OR (label = 'Different state' AND estimate = 86504176)\n",
    "OR (label = 'Born in Puerto Rico, U.S. Island areas, or born abroad to American parent(s)')\n",
    "OR (label = 'Foreign born' AND estimate = 44011870)\n",
    "OR (label = 'Foreign-born population')\n",
    "OR (label = 'Naturalized U.S. citizen')\n",
    "OR (label = 'Not a U.S. citizen ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "adaptive-momentum",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.social\n",
    "SET subcategory = 'year of entry'\n",
    "WHERE (label = 'Population born outside the United States')\n",
    "OR (label = 'Native' AND estimate = 5148655)\n",
    "OR (label = 'Foreign born' AND estimate = 44011870 AND percent = 13.6)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "funded-business",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "UPDATE datamart.social \n",
    "SET subcategory = 'region of birth of foreign born'\n",
    "WHERE label in ('Foreign-born population, excluding population born at sea', 'Europe', 'Asia', 'Africa', 'Oceania', 'Latin America', 'Northern America')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "durable-admission",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "UPDATE datamart.social \n",
    "SET subcategory = 'language spoken at home'\n",
    "WHERE label in ('Population 5 years and over', 'English only', 'Language other than English', 'Spanish', 'Other Indo-European languages', 'Asian and Pacific Islander languages', 'Other languages')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "intellectual-integral",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "UPDATE datamart.social\n",
    "SET subcategory = 'ancestry'\n",
    "WHERE label in ('Total population', 'American', 'Arab', 'Czech', 'Danish', 'Dutch', 'English', 'French (except Basque)', 'French Canadian', 'German', 'Greek', 'Hungarian', 'Irish', 'Italian', 'Lithuanian', 'Norwegian', 'Polish', 'Portuguese', 'Russian', 'Scotch-Irish', 'Scottish', 'Slovak', 'Subsaharan African', 'Swedish', 'Swiss', 'Ukrainian', 'Welsh', 'West Indian (excluding Hispanic origin groups)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "respected-chicken",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery \n",
    "UPDATE datamart.social\n",
    "SET subcategory = 'computer and internet use'\n",
    "WHERE label in ('With a computer', 'With a broadband Internet subscription')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cosmetic-chemistry",
   "metadata": {},
   "source": [
    "#### Make fields null in pecent col if they are not percentages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "atmospheric-construction",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "UPDATE datamart.social\n",
    "SET percent=null\n",
    "WHERE estimate=percent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "boolean-alfred",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE OR REPLACE TABLE datamart.social AS\n",
    "SELECT DISTINCT *\n",
    "FROM datamart.social"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "historical-syria",
   "metadata": {},
   "source": [
    "#### Check Primary Keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "liked-riding",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>distinct_PK</th>\n",
       "      <th>social_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>128</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   distinct_PK  social_count\n",
       "0          128           128"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT count(*) AS distinct_PK, (SELECT count(*) FROM datamart.social) AS social_count\n",
    "FROM (\n",
    "    SELECT DISTINCT category, subcategory, label\n",
    "    FROM datamart.social\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "contained-aside",
   "metadata": {},
   "source": [
    "#### Check FK violations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bibliographic-reviewer",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>foreign_key_violations</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   foreign_key_violations\n",
       "0                       0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT count(*) AS foreign_key_violations\n",
    "FROM datamart.social AS h\n",
    "LEFT JOIN datamart.locations AS l ON h.country=l.country\n",
    "WHERE l.id IS null"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "painted-opening",
   "metadata": {},
   "source": [
    "### SQL Queries"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "wrong-position",
   "metadata": {},
   "source": [
    "#### Get the estimated population of Americans affected by COVID-19 based on the average US April 14, 2021 incident rate, and display the estimated population distributed by the percentage of household income allocated to gross rent (assuming every American household occupies a housing unit for rent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "stopped-winning",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Percentage_of_Household_Income_for_Gross_Rent</th>\n",
       "      <th>Estimated_COVID_Affected_Population</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>15.0 to 19.9 percent</td>\n",
       "      <td>3.520428e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20.0 to 24.9 percent</td>\n",
       "      <td>3.520428e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>25.0 to 29.9 percent</td>\n",
       "      <td>3.165657e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>30.0 to 34.9 percent</td>\n",
       "      <td>2.483403e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>35.0 percent or more</td>\n",
       "      <td>1.105251e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Less than 15.0 percent</td>\n",
       "      <td>3.575009e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Percentage_of_Household_Income_for_Gross_Rent  \\\n",
       "0                          15.0 to 19.9 percent   \n",
       "1                          20.0 to 24.9 percent   \n",
       "2                          25.0 to 29.9 percent   \n",
       "3                          30.0 to 34.9 percent   \n",
       "4                          35.0 percent or more   \n",
       "5                        Less than 15.0 percent   \n",
       "\n",
       "   Estimated_COVID_Affected_Population  \n",
       "0                         3.520428e+06  \n",
       "1                         3.520428e+06  \n",
       "2                         3.165657e+06  \n",
       "3                         2.483403e+06  \n",
       "4                         1.105251e+07  \n",
       "5                         3.575009e+06  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT label AS Percentage_of_Household_Income_for_Gross_Rent, (h.percent/100)*\n",
    "                                (((SELECT AVG(incident_rate)\n",
    "                                    FROM datamart.cases \n",
    "                                    WHERE last_update=(SELECT MAX(last_update) \n",
    "                                                        FROM datamart.cases AS c \n",
    "                                                        JOIN datamart.locations AS l ON c.location_id=l.id \n",
    "                                                        WHERE l.country='US' AND last_update between '2021-04-14' and '2021-04-14 23:59:59'))/100000)\n",
    "                                *\n",
    "                                (SELECT estimate FROM datamart.demographics WHERE label='Total population')) AS Estimated_COVID_Affected_Population \n",
    "FROM datamart.housing AS h\n",
    "WHERE subcategory='gross rent as a percentage of household income' AND percent IS NOT null\n",
    "ORDER BY Percentage_of_Household_Income_for_Gross_Rent"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "french-bulgaria",
   "metadata": {},
   "source": [
    "#### Get the projected number of COVID-19 deaths in the US based on the US's average April 14, 2021 (our most recent data entry) case fatality ratio , and display the estimated deaths distributed by race (assuming that the projected number of deaths is accurately split by the race percentages reported in demographics [focused on the subcategories with one race/two or more races]; the same day's average incident rate is used to project the number of people affected by COVID-19)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "presidential-madonna",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Race</th>\n",
       "      <th>Projected_Deaths</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>American Indian and Alaska Native</td>\n",
       "      <td>4482.362154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Asian</td>\n",
       "      <td>30816.239806</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Black or African American</td>\n",
       "      <td>71157.499189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Black or African American and American Indian ...</td>\n",
       "      <td>560.295269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Native Hawaiian and Other Pacific Islander</td>\n",
       "      <td>1120.590538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Some other race</td>\n",
       "      <td>27454.468191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>White</td>\n",
       "      <td>406214.070172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>White and American Indian and Alaska Native</td>\n",
       "      <td>3361.771615</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>White and Asian</td>\n",
       "      <td>3922.066884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>White and Black or African American</td>\n",
       "      <td>5042.657423</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Race  Projected_Deaths\n",
       "0                  American Indian and Alaska Native       4482.362154\n",
       "1                                              Asian      30816.239806\n",
       "2                          Black or African American      71157.499189\n",
       "3  Black or African American and American Indian ...        560.295269\n",
       "4         Native Hawaiian and Other Pacific Islander       1120.590538\n",
       "5                                    Some other race      27454.468191\n",
       "6                                              White     406214.070172\n",
       "7        White and American Indian and Alaska Native       3361.771615\n",
       "8                                    White and Asian       3922.066884\n",
       "9                White and Black or African American       5042.657423"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT label AS Race, (d.percent/100)*(\n",
    "                                        ((SELECT AVG(case_fatality_ratio)\n",
    "                                        FROM datamart.cases\n",
    "                                        WHERE last_update=(\n",
    "                                        SELECT MAX(last_update)\n",
    "                                        FROM datamart.cases AS c \n",
    "                                        JOIN datamart.locations AS l ON c.location_id=l.id \n",
    "                                        WHERE l.country='US' AND last_update between '2021-04-14' and '2021-04-14 23:59:59'))/100)\n",
    "                                        *\n",
    "                                        (((SELECT AVG(incident_rate) \n",
    "                                        FROM datamart.cases\n",
    "                                        WHERE last_update=(\n",
    "                                        SELECT MAX(last_update)\n",
    "                                        FROM datamart.cases AS c \n",
    "                                        JOIN datamart.locations AS l ON c.location_id=l.id \n",
    "                                        WHERE l.country='US' AND last_update between '2021-04-14' and '2021-04-14 23:59:59'))/100000)*(SELECT estimate FROM datamart.demographics WHERE label='Total population'))\n",
    "                                        ) AS Projected_Deaths\n",
    "FROM datamart.demographics AS d\n",
    "WHERE (subcategory1='one race' AND label IN ('White', 'Black or African American', 'Asian', 'Native Hawaiian and Other Pacific Islander', 'American Indian and Alaska Native', 'Some other race'))\n",
    "OR (subcategory1='two or more races' AND label IN ('White and Black or African American', 'White and American Indian and Alaska Native', 'White and Asian', 'Black or African American and American Indian and Alaska Native'))\n",
    "ORDER BY Race"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "together-nickname",
   "metadata": {},
   "source": [
    "#### Get the number of Americans actively affected by COVID-19 on January 10, 2021 and display the number of Americans by race (assuming that the total number of active cases is accurately split by the race percentages reported in demographics [focused on the subcategories with Hispanic or Latino/Not Hispanic or Latino]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "middle-exposure",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Race</th>\n",
       "      <th>Active_COVID_Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>American Indian and Alaska Native alone</td>\n",
       "      <td>152345.802000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Asian alone</td>\n",
       "      <td>1197002.730000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Black or African American alone</td>\n",
       "      <td>2676933.378000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Cuban</td>\n",
       "      <td>152345.802000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Mexican</td>\n",
       "      <td>2437532.832000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Native Hawaiian and Other Pacific Islander alone</td>\n",
       "      <td>43527.372000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Other Hispanic or Latino</td>\n",
       "      <td>935838.498000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Puerto Rican</td>\n",
       "      <td>369982.662000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Some other race alone</td>\n",
       "      <td>43527.372000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Two or more races</td>\n",
       "      <td>522328.464000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>White alone</td>\n",
       "      <td>13210557.402000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Race  Active_COVID_Cases\n",
       "0            American Indian and Alaska Native alone    152345.802000000\n",
       "1                                        Asian alone   1197002.730000000\n",
       "2                    Black or African American alone   2676933.378000000\n",
       "3                                              Cuban    152345.802000000\n",
       "4                                            Mexican   2437532.832000000\n",
       "5   Native Hawaiian and Other Pacific Islander alone     43527.372000000\n",
       "6                           Other Hispanic or Latino    935838.498000000\n",
       "7                                       Puerto Rican    369982.662000000\n",
       "8                              Some other race alone     43527.372000000\n",
       "9                                  Two or more races    522328.464000000\n",
       "10                                       White alone  13210557.402000000"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT d.label AS Race, (d.percent/100)*\n",
    "                                        (SELECT SUM(active)\n",
    "                                        FROM datamart.cases AS c\n",
    "                                        JOIN datamart.locations AS l ON c.location_id=l.id\n",
    "                                        WHERE l.country='US' AND last_update between '2021-01-10' and '2021-01-10 23:59:59') AS Active_COVID_Cases\n",
    "FROM datamart.demographics AS d\n",
    "WHERE (subcategory1='Hispanic or Latino' AND label IN ('Mexican', 'Puerto Rican', 'Cuban', 'Other Hispanic or Latino'))\n",
    "OR (subcategory1='Not Hispanic or Latino' AND label IN ('White alone', 'Black or African American alone', 'American Indian and Alaska Native alone', 'Asian alone', 'Native Hawaiian and Other Pacific Islander alone', 'Some other race alone', 'Two or more races'))\n",
    "ORDER BY Race"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "operating-finance",
   "metadata": {},
   "source": [
    "#### Get the number of estimated confirmed cases of working Americans affected by COVID-19 in 2021 (Jan 1 - April 14) per income group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "conceptual-neighbor",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Income</th>\n",
       "      <th>Estimated_Confirmed_COVID_Cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>$50,000 to $74,999</td>\n",
       "      <td>74710568834605272.228000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>$100,000 to $149,999</td>\n",
       "      <td>65588929616426721.549000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>$75,000 to $99,999</td>\n",
       "      <td>55164199081365520.773000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>$35,000 to $49,999</td>\n",
       "      <td>53426743992188653.977000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>$25,000 to $34,999</td>\n",
       "      <td>38658375734185286.211000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>$15,000 to $24,999</td>\n",
       "      <td>38658375734185286.211000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>$200,000 or more</td>\n",
       "      <td>33446010466654685.823000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>$150,000 to $199,999</td>\n",
       "      <td>29536736516006735.532000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Less than $10,000</td>\n",
       "      <td>26061826337653001.940000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>$10,000 to $14,999</td>\n",
       "      <td>18677642208651318.057000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Income Estimated_Confirmed_COVID_Cases\n",
       "0    $50,000 to $74,999     74710568834605272.228000000\n",
       "1  $100,000 to $149,999     65588929616426721.549000000\n",
       "2    $75,000 to $99,999     55164199081365520.773000000\n",
       "3    $35,000 to $49,999     53426743992188653.977000000\n",
       "4    $25,000 to $34,999     38658375734185286.211000000\n",
       "5    $15,000 to $24,999     38658375734185286.211000000\n",
       "6      $200,000 or more     33446010466654685.823000000\n",
       "7  $150,000 to $199,999     29536736516006735.532000000\n",
       "8     Less than $10,000     26061826337653001.940000000\n",
       "9    $10,000 to $14,999     18677642208651318.057000000"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT e.label AS Income, (e.percent/100)*\n",
    "                                        ((SELECT SUM(confirmed)\n",
    "                                        FROM datamart.cases AS c\n",
    "                                        JOIN datamart.locations AS l ON c.location_id=l.id\n",
    "                                        WHERE l.country='US' AND last_update between '2021-01-01' and '2021-04-14 23:59:59')\n",
    "                                        *\n",
    "                                        (SELECT estimate FROM datamart.economics WHERE label = 'Workers 16 years and over')) AS Estimated_Confirmed_COVID_Cases\n",
    "FROM datamart.economics AS e\n",
    "WHERE (category = 'employment' AND subcategory= 'income')\n",
    "ORDER BY Estimated_Confirmed_COVID_Cases DESC"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "crucial-chair",
   "metadata": {},
   "source": [
    "#### Get the estimated number of deaths on December 31, 2020 from COVID-19 per different ancestry of US residents based on percentage of population's ancestry and total confirmed deaths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fewer-article",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Ancestry</th>\n",
       "      <th>Estimated_Deaths</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>American</td>\n",
       "      <td>2160439.600000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Arab</td>\n",
       "      <td>209074.800000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Czech</td>\n",
       "      <td>139383.200000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Danish</td>\n",
       "      <td>139383.200000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Dutch</td>\n",
       "      <td>418149.600000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>English</td>\n",
       "      <td>2508897.600000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>French (except Basque)</td>\n",
       "      <td>801453.400000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>French Canadian</td>\n",
       "      <td>209074.800000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>German</td>\n",
       "      <td>4634491.400000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Greek</td>\n",
       "      <td>139383.200000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Hungarian</td>\n",
       "      <td>139383.200000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Irish</td>\n",
       "      <td>3380042.600000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Italian</td>\n",
       "      <td>1777135.800000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Lithuanian</td>\n",
       "      <td>69691.600000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Norwegian</td>\n",
       "      <td>452995.400000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Polish</td>\n",
       "      <td>975682.400000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Portuguese</td>\n",
       "      <td>139383.200000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Russian</td>\n",
       "      <td>278766.400000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Scotch-Irish</td>\n",
       "      <td>313612.200000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Scottish</td>\n",
       "      <td>592378.600000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Slovak</td>\n",
       "      <td>69691.600000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Subsaharan African</td>\n",
       "      <td>418149.600000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Swedish</td>\n",
       "      <td>383303.800000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Swiss</td>\n",
       "      <td>104537.400000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Ukrainian</td>\n",
       "      <td>104537.400000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Welsh</td>\n",
       "      <td>209074.800000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>West Indian (excluding Hispanic origin groups)</td>\n",
       "      <td>313612.200000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          Ancestry   Estimated_Deaths\n",
       "0                                         American  2160439.600000000\n",
       "1                                             Arab   209074.800000000\n",
       "2                                            Czech   139383.200000000\n",
       "3                                           Danish   139383.200000000\n",
       "4                                            Dutch   418149.600000000\n",
       "5                                          English  2508897.600000000\n",
       "6                           French (except Basque)   801453.400000000\n",
       "7                                  French Canadian   209074.800000000\n",
       "8                                           German  4634491.400000000\n",
       "9                                            Greek   139383.200000000\n",
       "10                                       Hungarian   139383.200000000\n",
       "11                                           Irish  3380042.600000000\n",
       "12                                         Italian  1777135.800000000\n",
       "13                                      Lithuanian    69691.600000000\n",
       "14                                       Norwegian   452995.400000000\n",
       "15                                          Polish   975682.400000000\n",
       "16                                      Portuguese   139383.200000000\n",
       "17                                         Russian   278766.400000000\n",
       "18                                    Scotch-Irish   313612.200000000\n",
       "19                                        Scottish   592378.600000000\n",
       "20                                          Slovak    69691.600000000\n",
       "21                              Subsaharan African   418149.600000000\n",
       "22                                         Swedish   383303.800000000\n",
       "23                                           Swiss   104537.400000000\n",
       "24                                       Ukrainian   104537.400000000\n",
       "25                                           Welsh   209074.800000000\n",
       "26  West Indian (excluding Hispanic origin groups)   313612.200000000"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "SELECT s.label AS Ancestry, (s.percent)*\n",
    "                                        (SELECT SUM(deaths)\n",
    "                                        FROM datamart.cases AS c\n",
    "                                        JOIN datamart.locations AS l ON c.location_id=l.id\n",
    "                                        WHERE l.country='US' AND last_update between '2020-12-31' and '2020-12-31 23:59:59') AS Estimated_Deaths\n",
    "                                    \n",
    "FROM datamart.social as s\n",
    "WHERE (subcategory = 'ancestry')\n",
    "ORDER BY Ancestry"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "loving-words",
   "metadata": {},
   "source": [
    "### Views"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "light-seminar",
   "metadata": {},
   "source": [
    "#### Create BigQuery Dataset \"reports\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "unavailable-burlington",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset 'lunar-analyzer-302702:reports' successfully created.\n"
     ]
    }
   ],
   "source": [
    "!bq --location=US mk --dataset reports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "attractive-acting",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE VIEW reports.estimated_covid_population_by_income_for_gross_rent AS\n",
    "SELECT label AS Percentage_of_Household_Income_for_Gross_Rent, (h.percent/100)*\n",
    "                                (((SELECT AVG(incident_rate)\n",
    "                                    FROM datamart.cases \n",
    "                                    WHERE last_update=(SELECT MAX(last_update) \n",
    "                                                        FROM datamart.cases AS c \n",
    "                                                        JOIN datamart.locations AS l ON c.location_id=l.id \n",
    "                                                        WHERE l.country='US' AND last_update between '2021-04-14' and '2021-04-14 23:59:59'))/100000)\n",
    "                                *\n",
    "                                (SELECT estimate FROM datamart.demographics WHERE label='Total population')) AS Estimated_COVID_Affected_Population \n",
    "FROM datamart.housing AS h\n",
    "WHERE subcategory='gross rent as a percentage of household income' AND percent IS NOT null\n",
    "ORDER BY Percentage_of_Household_Income_for_Gross_Rent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "accepting-shelter",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%bigquery\n",
    "CREATE VIEW reports.estimated_deaths_by_ancestry_dec3120 AS\n",
    "SELECT s.label AS Ancestry, (s.percent)*\n",
    "                                        (SELECT SUM(deaths)\n",
    "                                        FROM datamart.cases AS c\n",
    "                                        JOIN datamart.locations AS l ON c.location_id=l.id\n",
    "                                        WHERE l.country='US' AND last_update between '2020-12-31' and '2020-12-31 23:59:59') AS Estimated_Deaths\n",
    "                                    \n",
    "FROM datamart.social as s\n",
    "WHERE (subcategory = 'ancestry')\n",
    "ORDER BY Ancestry"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sapphire-nothing",
   "metadata": {},
   "source": [
    "### Temporary tables were deleted via BigQuery's UI"
   ]
  }
 ],
 "metadata": {
  "environment": {
   "name": "common-cpu.mnightly-2021-01-20-debian-10-test",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/base-cpu:mnightly-2021-01-20-debian-10-test"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}