WaterVaporNN_1_DataOrganization_1_TestData.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.display import HTML, display\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extracting and Organizing Training Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# column water vapor lat 361 (0.5 degrees)  x lon 576 (0.625 degrees) (cm)\n",
    "cwv_merra = np.load('./TestingData/WV_val.npy')\n",
    "# delta_pressure (atm) (19x361x576 vertical layers; 0 is surface, 18 is )\n",
    "dp_merra = np.load('./TestingData/DP_val.npy')\n",
    "# Volume Mixing Ratio () (19x361x576)\n",
    "vmrm_merra = np.load('./TestingData/VMRM_val.npy')\n",
    "#MODISA/MERRA Water Vapor Transmittance (unitless) (16x361x576)\n",
    "h20_g_trans = np.load('./TestingData/WV_tbl_MERRA_1_val.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((361, 576), (19, 361, 576), (19, 361, 576), (16, 361, 200))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cwv_merra.shape, dp_merra.shape, vmrm_merra.shape, h20_g_trans.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "lat = np.linspace(-90, 90, num=361)\n",
    "lon = np.linspace(-180+0.625, 180, num=576)\n",
    "layers = [f'vl_{i}' for i in range(19)]\n",
    "wavelengths = ['412', '443', '469', '488', '531', '547', '555', '645', '667',\n",
    "              '678', '748', '859', '869', '1240', '1640', '2130']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "lon_mesh, lat_mesh = np.meshgrid(lon, lat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "lon_mesh = lon_mesh.reshape((lon_mesh.size,))\n",
    "lat_mesh = lat_mesh.reshape((lat_mesh.size,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "cannot reshape array of size 1155200 into shape (207936)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-e2d54ebe579d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mdp_merra_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdp_merra\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlon_mesh\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mvmrm_merra_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvmrm_merra\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlon_mesh\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mh20_g_trans_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh20_g_trans\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlon_mesh\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m: cannot reshape array of size 1155200 into shape (207936)"
     ]
    }
   ],
   "source": [
    "cwv_merra_ = cwv_merra.reshape(lon_mesh.shape)\n",
    "dp_merra_ = dp_merra.reshape((-1, lon_mesh.size)).T\n",
    "vmrm_merra_ = vmrm_merra.reshape((-1, lon_mesh.size)).T\n",
    "h20_g_trans_ = h20_g_trans.reshape((-1, lon_mesh.size)).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(207936,)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cwv_merra_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "lon_lat_cwv_cols = ['lon', 'lat', 'cwv']\n",
    "dp_cols = [f'dp_{vli}' for vli in layers]\n",
    "vmrm_cols = [f'vmr_{vli}' for vli in layers]\n",
    "wvt_cols = [f'wvt_{wvi}' for wvi in wavelengths]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(np.c_[lon_mesh, lat_mesh, cwv_merra_, dp_merra_, vmrm_merra_, h20_g_trans_],\n",
    "                  columns = lon_lat_cwv_cols + dp_cols + vmrm_cols + wvt_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 138927 entries, 7523 to 207935\n",
      "Data columns (total 57 columns):\n",
      "lon          138927 non-null float64\n",
      "lat          138927 non-null float64\n",
      "cwv          138927 non-null float64\n",
      "dp_vl_0      138927 non-null float64\n",
      "dp_vl_1      138927 non-null float64\n",
      "dp_vl_2      138927 non-null float64\n",
      "dp_vl_3      138927 non-null float64\n",
      "dp_vl_4      138927 non-null float64\n",
      "dp_vl_5      138927 non-null float64\n",
      "dp_vl_6      138927 non-null float64\n",
      "dp_vl_7      138927 non-null float64\n",
      "dp_vl_8      138927 non-null float64\n",
      "dp_vl_9      138927 non-null float64\n",
      "dp_vl_10     138927 non-null float64\n",
      "dp_vl_11     138927 non-null float64\n",
      "dp_vl_12     138927 non-null float64\n",
      "dp_vl_13     138927 non-null float64\n",
      "dp_vl_14     138927 non-null float64\n",
      "dp_vl_15     138927 non-null float64\n",
      "dp_vl_16     138927 non-null float64\n",
      "dp_vl_17     138927 non-null float64\n",
      "dp_vl_18     138927 non-null float64\n",
      "vmr_vl_0     138927 non-null float64\n",
      "vmr_vl_1     138927 non-null float64\n",
      "vmr_vl_2     138927 non-null float64\n",
      "vmr_vl_3     138927 non-null float64\n",
      "vmr_vl_4     138927 non-null float64\n",
      "vmr_vl_5     138927 non-null float64\n",
      "vmr_vl_6     138927 non-null float64\n",
      "vmr_vl_7     138927 non-null float64\n",
      "vmr_vl_8     138927 non-null float64\n",
      "vmr_vl_9     138927 non-null float64\n",
      "vmr_vl_10    138927 non-null float64\n",
      "vmr_vl_11    138927 non-null float64\n",
      "vmr_vl_12    138927 non-null float64\n",
      "vmr_vl_13    138927 non-null float64\n",
      "vmr_vl_14    138927 non-null float64\n",
      "vmr_vl_15    138927 non-null float64\n",
      "vmr_vl_16    138927 non-null float64\n",
      "vmr_vl_17    138927 non-null float64\n",
      "vmr_vl_18    138927 non-null float64\n",
      "wvt_412      138927 non-null float64\n",
      "wvt_443      138927 non-null float64\n",
      "wvt_469      138927 non-null float64\n",
      "wvt_488      138927 non-null float64\n",
      "wvt_531      138927 non-null float64\n",
      "wvt_547      138927 non-null float64\n",
      "wvt_555      138927 non-null float64\n",
      "wvt_645      138927 non-null float64\n",
      "wvt_667      138927 non-null float64\n",
      "wvt_678      138927 non-null float64\n",
      "wvt_748      138927 non-null float64\n",
      "wvt_859      138927 non-null float64\n",
      "wvt_869      138927 non-null float64\n",
      "wvt_1240     138927 non-null float64\n",
      "wvt_1640     138927 non-null float64\n",
      "wvt_2130     138927 non-null float64\n",
      "dtypes: float64(57)\n",
      "memory usage: 61.5 MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_pickle('./pickleJar/testing.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:machine_learning]",
   "language": "python",
   "name": "conda-env-machine_learning-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}