convert_gps_csv.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import calendar\n",
    "import datetime\n",
    "#import dateutil.parser\n",
    "import gpxpy \n",
    "import gpxpy.gpx\n",
    "import os\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import pendulum\n",
    "import pydarksky\n",
    "import numpy as np\n",
    "import requests\n",
    "import sys\n",
    "import time\n",
    "import timeit\n",
    "\n",
    "from glob import glob\n",
    "from pathlib import Path\n",
    "\n",
    "#from xml.dom import minidom\n",
    "#from gpx_csv_converter import Converter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setup directory for local libraty Import in notebook\n",
    "current_dir = get_ipython().getoutput('pwd')\n",
    "module_path = Path(current_dir[0]).parent\n",
    "\n",
    "if module_path not in sys.path:\n",
    "    sys.path.append(str(module_path))\n",
    "\n",
    "# importy Modified gpx_csv_converter CLASS\n",
    "from gpx_csv_convert import Converter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " path from gpx:\n",
      " ['gpx\\\\running\\\\Snow_corridor_Street_porridge.gpx', 'gpx\\\\running\\\\Snow_waist_Water_under_snow_Hard_snow.gpx', 'gpx\\\\running\\\\Some_intervals_in_snow_Frosty_roads.gpx']\n",
      "\n",
      " path to csv: \n",
      " ['csv\\\\running\\\\Snow_corridor_Street_porridge.csv', 'csv\\\\running\\\\Snow_waist_Water_under_snow_Hard_snow.csv', 'csv\\\\running\\\\Some_intervals_in_snow_Frosty_roads.csv']\n"
     ]
    }
   ],
   "source": [
    "# Path to data - files operations\n",
    "\n",
    "PATH_TO_DATA = os.path.abspath(os.curdir)\n",
    "path_to_files = 'gpx\\\\running\\\\*'\n",
    "\n",
    "# File operations\n",
    "file_quant = len(glob(path_to_files))\n",
    "file_names = [file for file in glob(path_to_files)]\n",
    "file_length = len(file_names)\n",
    "\n",
    "#files_gpx = os.listdir('gpx')\n",
    "#files_gpx = [('gpx\\\\' + file) for file in os.listdir('gpx')]\n",
    "files_csv = [ ('csv' + path_to_files[3:-1] + file[:len(file) - 4] + '.csv') for file in os.listdir(path_to_files[:-2])]\n",
    "\n",
    "print(' path from gpx:\\n',file_names)\n",
    "print('\\n path to csv: \\n',files_csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "file: 1 -> gpx\\running\\Snow_corridor_Street_porridge.gpx  time: 4 s \n",
      "\n",
      "file: 2 -> gpx\\running\\Snow_waist_Water_under_snow_Hard_snow.gpx  time: 5 s \n",
      "\n",
      "file: 3 -> gpx\\running\\Some_intervals_in_snow_Frosty_roads.gpx  time: 4 s \n",
      "\n",
      "Wall time: 13.1 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# OPEN GPS file for XML parse (modified libruary)\n",
    "\n",
    "# gpx_file_xml = []\n",
    "\n",
    "# convert gpx -> csv\n",
    "for ind, (csv, file) in enumerate(zip(files_csv, file_names)):\n",
    "    \n",
    "    start_time = timeit.default_timer()\n",
    "    \n",
    "    gpx_file = open( file, 'r' )\n",
    "    gpx_file_str = gpx_file.read()\n",
    "    \n",
    "    # parse gps by xml tags\n",
    "    Converter(gpx_file_str, csv[:len(csv) - 4])\n",
    "   \n",
    "    gpx_file.close()\n",
    "    \n",
    "    end_time = timeit.default_timer() - start_time\n",
    "    \n",
    "    print(f'file: {ind + 1} -> {file}  time: {end_time:.0f} s \\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data_time</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>elevation</th>\n",
       "      <th>heart_rate</th>\n",
       "      <th>cadence</th>\n",
       "      <th>temperature</th>\n",
       "      <th>sport_type</th>\n",
       "      <th>id</th>\n",
       "      <th>athlete</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019-01-28 13:56:10</td>\n",
       "      <td>49.409331</td>\n",
       "      <td>32.097608</td>\n",
       "      <td>90.4</td>\n",
       "      <td>97</td>\n",
       "      <td>61.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>running</td>\n",
       "      <td>philka_ua</td>\n",
       "      <td>amateur</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019-01-28 13:56:10</td>\n",
       "      <td>49.409345</td>\n",
       "      <td>32.097593</td>\n",
       "      <td>90.4</td>\n",
       "      <td>98</td>\n",
       "      <td>61.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>running</td>\n",
       "      <td>philka_ua</td>\n",
       "      <td>amateur</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2019-01-28 13:56:11</td>\n",
       "      <td>49.409358</td>\n",
       "      <td>32.097570</td>\n",
       "      <td>90.2</td>\n",
       "      <td>98</td>\n",
       "      <td>61.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>running</td>\n",
       "      <td>philka_ua</td>\n",
       "      <td>amateur</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2019-01-28 13:56:12</td>\n",
       "      <td>49.409369</td>\n",
       "      <td>32.097546</td>\n",
       "      <td>90.2</td>\n",
       "      <td>97</td>\n",
       "      <td>61.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>running</td>\n",
       "      <td>philka_ua</td>\n",
       "      <td>amateur</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019-01-28 13:56:13</td>\n",
       "      <td>49.409380</td>\n",
       "      <td>32.097521</td>\n",
       "      <td>90.2</td>\n",
       "      <td>97</td>\n",
       "      <td>61.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>running</td>\n",
       "      <td>philka_ua</td>\n",
       "      <td>amateur</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2019-01-28 13:56:14</td>\n",
       "      <td>49.409392</td>\n",
       "      <td>32.097495</td>\n",
       "      <td>89.8</td>\n",
       "      <td>97</td>\n",
       "      <td>85.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>running</td>\n",
       "      <td>philka_ua</td>\n",
       "      <td>amateur</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2019-01-28 13:56:15</td>\n",
       "      <td>49.409405</td>\n",
       "      <td>32.097469</td>\n",
       "      <td>89.8</td>\n",
       "      <td>97</td>\n",
       "      <td>85.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>running</td>\n",
       "      <td>philka_ua</td>\n",
       "      <td>amateur</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2019-01-28 13:56:16</td>\n",
       "      <td>49.409411</td>\n",
       "      <td>32.097439</td>\n",
       "      <td>89.3</td>\n",
       "      <td>98</td>\n",
       "      <td>85.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>running</td>\n",
       "      <td>philka_ua</td>\n",
       "      <td>amateur</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2019-01-28 13:56:17</td>\n",
       "      <td>49.409413</td>\n",
       "      <td>32.097411</td>\n",
       "      <td>89.1</td>\n",
       "      <td>98</td>\n",
       "      <td>84.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>running</td>\n",
       "      <td>philka_ua</td>\n",
       "      <td>amateur</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2019-01-28 13:56:18</td>\n",
       "      <td>49.409405</td>\n",
       "      <td>32.097384</td>\n",
       "      <td>89.3</td>\n",
       "      <td>98</td>\n",
       "      <td>83.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>running</td>\n",
       "      <td>philka_ua</td>\n",
       "      <td>amateur</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            data_time   latitude  longitude  elevation  heart_rate  cadence  \\\n",
       "0 2019-01-28 13:56:10  49.409331  32.097608       90.4          97     61.0   \n",
       "1 2019-01-28 13:56:10  49.409345  32.097593       90.4          98     61.0   \n",
       "2 2019-01-28 13:56:11  49.409358  32.097570       90.2          98     61.0   \n",
       "3 2019-01-28 13:56:12  49.409369  32.097546       90.2          97     61.0   \n",
       "4 2019-01-28 13:56:13  49.409380  32.097521       90.2          97     61.0   \n",
       "5 2019-01-28 13:56:14  49.409392  32.097495       89.8          97     85.0   \n",
       "6 2019-01-28 13:56:15  49.409405  32.097469       89.8          97     85.0   \n",
       "7 2019-01-28 13:56:16  49.409411  32.097439       89.3          98     85.0   \n",
       "8 2019-01-28 13:56:17  49.409413  32.097411       89.1          98     84.0   \n",
       "9 2019-01-28 13:56:18  49.409405  32.097384       89.3          98     83.0   \n",
       "\n",
       "   temperature sport_type         id  athlete  \n",
       "0          NaN    running  philka_ua  amateur  \n",
       "1          NaN    running  philka_ua  amateur  \n",
       "2          NaN    running  philka_ua  amateur  \n",
       "3          NaN    running  philka_ua  amateur  \n",
       "4          NaN    running  philka_ua  amateur  \n",
       "5          NaN    running  philka_ua  amateur  \n",
       "6          NaN    running  philka_ua  amateur  \n",
       "7          NaN    running  philka_ua  amateur  \n",
       "8          NaN    running  philka_ua  amateur  \n",
       "9          NaN    running  philka_ua  amateur  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import converted csv files\n",
    "user_data = [pd.read_csv(os.path.join(PATH_TO_DATA,file)) for file in files_csv]\n",
    "\n",
    "# Convert time and date to timestamp format (All files)\n",
    "for data in user_data:\n",
    "    # add datatime type\n",
    "    data.insert(2,'data_time',(data['date'] + ' ' + data['time'])\\\n",
    "        .apply(lambda x: datetime.datetime.strptime(x, \"%Y-%m-%d %H:%M:%S\" )))\n",
    "    # drop str time adn dates\n",
    "    data.drop(['date','time'], axis=1, inplace=True)\n",
    "    # add sport type (target)\n",
    "    data['sport_type'] = path_to_files[4:-2]\n",
    "    data['id'] = 'philka_ua'\n",
    "    data['athlete'] = 'amateur'\n",
    "    \n",
    "user_data[0].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}