WaterVaporNN_1_DataOrganization_1_TrainingData.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.display import HTML, display\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container {width:90% !important}</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(HTML(\"<style>.container {width:90% !important}</style>\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extracting and Organizing Training Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# column water vapor lat 361 (0.5 degrees)  x lon 576 (0.625 degrees) (cm)\n",
    "cwv_merra = np.load('./TrainingData/CWV_merra.npy')\n",
    "# delta_pressure (atm) (19x361x576 vertical layers; 0 is surface, 18 is )\n",
    "dp_merra = np.load('./TrainingData/DP_merra.npy')\n",
    "# Volume Mixing Ratio () (19x361x576)\n",
    "vmrm_merra = np.load('./TrainingData/VMRM_merra.npy')\n",
    "#MODISA/MERRA Water Vapor Transmittance (unitless) (16x361x576)\n",
    "h20_g_trans = np.load('./TrainingData/MODISA_MERRA_wv_tbl.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((361, 576), (19, 361, 576), (19, 361, 576), (16, 361, 576))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cwv_merra.shape, dp_merra.shape, vmrm_merra.shape, h20_g_trans.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "lat = np.linspace(-90, 90, num=361)\n",
    "lon = np.linspace(-180+0.625, 180, num=576)\n",
    "layers = [f'vl_{i}' for i in range(19)]\n",
    "wavelengths = ['412', '443', '469', '488', '531', '547', '555', '645', '667',\n",
    "              '678', '748', '859', '869', '1240', '1640', '2130']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "lon_mesh, lat_mesh = np.meshgrid(lon, lat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "lon_mesh = lon_mesh.reshape((lon_mesh.size,))\n",
    "lat_mesh = lat_mesh.reshape((lat_mesh.size,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "cwv_merra_ = cwv_merra.reshape(lon_mesh.shape)\n",
    "dp_merra_ = dp_merra.reshape((-1, lon_mesh.size)).T\n",
    "vmrm_merra_ = vmrm_merra.reshape((-1, lon_mesh.size)).T\n",
    "h20_g_trans_ = h20_g_trans.reshape((-1, lon_mesh.size)).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(207936, 16)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h20_g_trans_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(207936, 19)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dp_merra_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(207936, 57)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(207936,)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cwv_merra_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "lon_lat_cwv_cols = ['lon', 'lat', 'cwv']\n",
    "dp_cols = [f'dp_{vli}' for vli in layers]\n",
    "vmrm_cols = [f'vmr_{vli}' for vli in layers]\n",
    "wvt_cols = [f'wvt_{wvi}' for wvi in wavelengths]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(np.c_[lon_mesh, lat_mesh, cwv_merra_, dp_merra_, vmrm_merra_, h20_g_trans_],\n",
    "                  columns = lon_lat_cwv_cols + dp_cols + vmrm_cols + wvt_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>lon</th>\n",
       "      <td>-179.375000</td>\n",
       "      <td>-178.750000</td>\n",
       "      <td>-178.125000</td>\n",
       "      <td>-177.500000</td>\n",
       "      <td>-176.875000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lat</th>\n",
       "      <td>-90.000000</td>\n",
       "      <td>-90.000000</td>\n",
       "      <td>-90.000000</td>\n",
       "      <td>-90.000000</td>\n",
       "      <td>-90.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cwv</th>\n",
       "      <td>0.087462</td>\n",
       "      <td>0.087462</td>\n",
       "      <td>0.087462</td>\n",
       "      <td>0.087462</td>\n",
       "      <td>0.087462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dp_vl_0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dp_vl_1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dp_vl_2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dp_vl_3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dp_vl_4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dp_vl_5</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dp_vl_6</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dp_vl_7</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dp_vl_8</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dp_vl_9</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dp_vl_10</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dp_vl_11</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dp_vl_12</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dp_vl_13</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dp_vl_14</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dp_vl_15</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dp_vl_16</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dp_vl_17</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dp_vl_18</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vmr_vl_0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vmr_vl_1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vmr_vl_2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vmr_vl_3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vmr_vl_4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vmr_vl_5</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vmr_vl_6</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vmr_vl_7</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vmr_vl_8</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vmr_vl_9</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vmr_vl_10</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vmr_vl_11</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vmr_vl_12</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vmr_vl_13</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vmr_vl_14</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vmr_vl_15</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vmr_vl_16</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vmr_vl_17</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vmr_vl_18</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wvt_412</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wvt_443</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wvt_469</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wvt_488</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wvt_531</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wvt_547</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wvt_555</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wvt_645</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wvt_667</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wvt_678</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wvt_748</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wvt_859</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wvt_869</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wvt_1240</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wvt_1640</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wvt_2130</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    0           1           2           3           4\n",
       "lon       -179.375000 -178.750000 -178.125000 -177.500000 -176.875000\n",
       "lat        -90.000000  -90.000000  -90.000000  -90.000000  -90.000000\n",
       "cwv          0.087462    0.087462    0.087462    0.087462    0.087462\n",
       "dp_vl_0           NaN         NaN         NaN         NaN         NaN\n",
       "dp_vl_1           NaN         NaN         NaN         NaN         NaN\n",
       "dp_vl_2           NaN         NaN         NaN         NaN         NaN\n",
       "dp_vl_3           NaN         NaN         NaN         NaN         NaN\n",
       "dp_vl_4           NaN         NaN         NaN         NaN         NaN\n",
       "dp_vl_5           NaN         NaN         NaN         NaN         NaN\n",
       "dp_vl_6           NaN         NaN         NaN         NaN         NaN\n",
       "dp_vl_7           NaN         NaN         NaN         NaN         NaN\n",
       "dp_vl_8           NaN         NaN         NaN         NaN         NaN\n",
       "dp_vl_9           NaN         NaN         NaN         NaN         NaN\n",
       "dp_vl_10          NaN         NaN         NaN         NaN         NaN\n",
       "dp_vl_11          NaN         NaN         NaN         NaN         NaN\n",
       "dp_vl_12          NaN         NaN         NaN         NaN         NaN\n",
       "dp_vl_13          NaN         NaN         NaN         NaN         NaN\n",
       "dp_vl_14          NaN         NaN         NaN         NaN         NaN\n",
       "dp_vl_15          NaN         NaN         NaN         NaN         NaN\n",
       "dp_vl_16          NaN         NaN         NaN         NaN         NaN\n",
       "dp_vl_17          NaN         NaN         NaN         NaN         NaN\n",
       "dp_vl_18          NaN         NaN         NaN         NaN         NaN\n",
       "vmr_vl_0          NaN         NaN         NaN         NaN         NaN\n",
       "vmr_vl_1          NaN         NaN         NaN         NaN         NaN\n",
       "vmr_vl_2          NaN         NaN         NaN         NaN         NaN\n",
       "vmr_vl_3          NaN         NaN         NaN         NaN         NaN\n",
       "vmr_vl_4          NaN         NaN         NaN         NaN         NaN\n",
       "vmr_vl_5          NaN         NaN         NaN         NaN         NaN\n",
       "vmr_vl_6          NaN         NaN         NaN         NaN         NaN\n",
       "vmr_vl_7          NaN         NaN         NaN         NaN         NaN\n",
       "vmr_vl_8          NaN         NaN         NaN         NaN         NaN\n",
       "vmr_vl_9          NaN         NaN         NaN         NaN         NaN\n",
       "vmr_vl_10         NaN         NaN         NaN         NaN         NaN\n",
       "vmr_vl_11         NaN         NaN         NaN         NaN         NaN\n",
       "vmr_vl_12         NaN         NaN         NaN         NaN         NaN\n",
       "vmr_vl_13         NaN         NaN         NaN         NaN         NaN\n",
       "vmr_vl_14         NaN         NaN         NaN         NaN         NaN\n",
       "vmr_vl_15         NaN         NaN         NaN         NaN         NaN\n",
       "vmr_vl_16         NaN         NaN         NaN         NaN         NaN\n",
       "vmr_vl_17         NaN         NaN         NaN         NaN         NaN\n",
       "vmr_vl_18         NaN         NaN         NaN         NaN         NaN\n",
       "wvt_412           NaN         NaN         NaN         NaN         NaN\n",
       "wvt_443           NaN         NaN         NaN         NaN         NaN\n",
       "wvt_469           NaN         NaN         NaN         NaN         NaN\n",
       "wvt_488           NaN         NaN         NaN         NaN         NaN\n",
       "wvt_531           NaN         NaN         NaN         NaN         NaN\n",
       "wvt_547           NaN         NaN         NaN         NaN         NaN\n",
       "wvt_555           NaN         NaN         NaN         NaN         NaN\n",
       "wvt_645           NaN         NaN         NaN         NaN         NaN\n",
       "wvt_667           NaN         NaN         NaN         NaN         NaN\n",
       "wvt_678           NaN         NaN         NaN         NaN         NaN\n",
       "wvt_748           NaN         NaN         NaN         NaN         NaN\n",
       "wvt_859           NaN         NaN         NaN         NaN         NaN\n",
       "wvt_869           NaN         NaN         NaN         NaN         NaN\n",
       "wvt_1240          NaN         NaN         NaN         NaN         NaN\n",
       "wvt_1640          NaN         NaN         NaN         NaN         NaN\n",
       "wvt_2130          NaN         NaN         NaN         NaN         NaN"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 138927 entries, 7523 to 207935\n",
      "Data columns (total 57 columns):\n",
      "lon          138927 non-null float64\n",
      "lat          138927 non-null float64\n",
      "cwv          138927 non-null float64\n",
      "dp_vl_0      138927 non-null float64\n",
      "dp_vl_1      138927 non-null float64\n",
      "dp_vl_2      138927 non-null float64\n",
      "dp_vl_3      138927 non-null float64\n",
      "dp_vl_4      138927 non-null float64\n",
      "dp_vl_5      138927 non-null float64\n",
      "dp_vl_6      138927 non-null float64\n",
      "dp_vl_7      138927 non-null float64\n",
      "dp_vl_8      138927 non-null float64\n",
      "dp_vl_9      138927 non-null float64\n",
      "dp_vl_10     138927 non-null float64\n",
      "dp_vl_11     138927 non-null float64\n",
      "dp_vl_12     138927 non-null float64\n",
      "dp_vl_13     138927 non-null float64\n",
      "dp_vl_14     138927 non-null float64\n",
      "dp_vl_15     138927 non-null float64\n",
      "dp_vl_16     138927 non-null float64\n",
      "dp_vl_17     138927 non-null float64\n",
      "dp_vl_18     138927 non-null float64\n",
      "vmr_vl_0     138927 non-null float64\n",
      "vmr_vl_1     138927 non-null float64\n",
      "vmr_vl_2     138927 non-null float64\n",
      "vmr_vl_3     138927 non-null float64\n",
      "vmr_vl_4     138927 non-null float64\n",
      "vmr_vl_5     138927 non-null float64\n",
      "vmr_vl_6     138927 non-null float64\n",
      "vmr_vl_7     138927 non-null float64\n",
      "vmr_vl_8     138927 non-null float64\n",
      "vmr_vl_9     138927 non-null float64\n",
      "vmr_vl_10    138927 non-null float64\n",
      "vmr_vl_11    138927 non-null float64\n",
      "vmr_vl_12    138927 non-null float64\n",
      "vmr_vl_13    138927 non-null float64\n",
      "vmr_vl_14    138927 non-null float64\n",
      "vmr_vl_15    138927 non-null float64\n",
      "vmr_vl_16    138927 non-null float64\n",
      "vmr_vl_17    138927 non-null float64\n",
      "vmr_vl_18    138927 non-null float64\n",
      "wvt_412      138927 non-null float64\n",
      "wvt_443      138927 non-null float64\n",
      "wvt_469      138927 non-null float64\n",
      "wvt_488      138927 non-null float64\n",
      "wvt_531      138927 non-null float64\n",
      "wvt_547      138927 non-null float64\n",
      "wvt_555      138927 non-null float64\n",
      "wvt_645      138927 non-null float64\n",
      "wvt_667      138927 non-null float64\n",
      "wvt_678      138927 non-null float64\n",
      "wvt_748      138927 non-null float64\n",
      "wvt_859      138927 non-null float64\n",
      "wvt_869      138927 non-null float64\n",
      "wvt_1240     138927 non-null float64\n",
      "wvt_1640     138927 non-null float64\n",
      "wvt_2130     138927 non-null float64\n",
      "dtypes: float64(57)\n",
      "memory usage: 61.5 MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_pickle('./pickleJar/training.pkl')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:machine_learning]",
   "language": "python",
   "name": "conda-env-machine_learning-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}