data_pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import subprocess\n",
    "from openpilot_tools.lib.logreader import LogReader\n",
    "from openpilot_tools.lib.framereader import FrameReader\n",
    "from scipy.signal import resample\n",
    "import numpy as np\n",
    "import cv2 as cv\n",
    "from skimage.transform import resize\n",
    "import h5py\n",
    "import time\n",
    "import shutil\n",
    "import matplotlib.pyplot as plt\n",
    "from PIL import Image, ImageEnhance\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessing_dir = \"./custom_data/preprocessing\"\n",
    "downscaled_dir = \"./custom_data/downscaled\"\n",
    "raw_videos = os.listdir(preprocessing_dir)\n",
    "resize_size = (50,160,2)\n",
    "resize_frame_size = (50,160,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_order_clips(raw_videos, clip_ends = 2):\n",
    "    video_sets = {}\n",
    "    clipped_videos = []\n",
    "    for video in raw_videos:\n",
    "        date_time = \"--\".join(video.split(\"--\")[0:2])\n",
    "        if date_time not in video_sets:\n",
    "            video_sets[date_time] = 0\n",
    "        video_sets[date_time] += 1\n",
    "        \n",
    "    for date_time, num_frags in sorted(video_sets.items()):\n",
    "        for frag in range(num_frags):\n",
    "            if (frag > clip_ends) and (frag < (num_frags - clip_ends)):\n",
    "                video_loc = date_time + \"--\" + str(frag)\n",
    "                clipped_videos.append(video_loc)\n",
    "    return clipped_videos\n",
    "clipped_videos = filter_order_clips(raw_videos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_logs(video_dir, frame_shape):\n",
    "    lr = list(LogReader(video_dir + \"/rlog.bz2\"))\n",
    "    speed_data = [l.carState.vEgo for l in lr if l.which() == 'carState']\n",
    "    speed_data = np.array(speed_data)\n",
    "    resampled_speeds = resample(speed_data, frame_shape)\n",
    "    return resampled_speeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_video(downscaled_dir, video_dir):\n",
    "    downscaled_vid = downscaled_dir + '/' + video_dir.split(\"-\")[-1] + str(time.time()) + 'preprocessed.mp4'\n",
    "    subprocess.call('ffmpeg -r 24 -i ' +  video_dir + '/fcamera.hevc' + \n",
    "                    ' -c:v libx265 -r 20 -filter:v scale=640:480 -crf 10 -c:a -i ' +\n",
    "                    downscaled_vid, shell = True)\n",
    "    return downscaled_vid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def opticalFlowDense(image_current, image_next):\n",
    "    \"\"\"\n",
    "    Args:\n",
    "        image_current : RGB image\n",
    "        image_next : RGB image\n",
    "    return:\n",
    "        optical flow magnitude and angle and stacked in a matrix\n",
    "    \"\"\"\n",
    "    image_current = np.array(image_current)\n",
    "    image_next = np.array(image_next)\n",
    "    gray_current = cv.cvtColor(image_current, cv.COLOR_RGB2GRAY)\n",
    "    gray_next = cv.cvtColor(image_next, cv.COLOR_RGB2GRAY)\n",
    "    flow = cv.calcOpticalFlowFarneback(gray_current, gray_next, None, 0.5, 1, 15, 2, 5, 1.3, 0)\n",
    "    return flow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def augment(image_current, image_next):\n",
    "    brightness = np.random.uniform(0.5, 1.5)\n",
    "    img1 = ImageEnhance.Brightness(image_current).enhance(brightness)\n",
    "    img2 = ImageEnhance.Brightness(image_next).enhance(brightness)            \n",
    "\n",
    "    color = np.random.uniform(0.5, 1.5)\n",
    "    img1 = ImageEnhance.Brightness(img1).enhance(color)\n",
    "    img2 = ImageEnhance.Brightness(img2).enhance(color)\n",
    "    return img1, img2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "def op_flow_video(preprocessed_video, augment_frames = True):\n",
    "    op_flows = []\n",
    "    frames = []\n",
    "    count = 0\n",
    "    vidcap = cv.VideoCapture(preprocessed_video)\n",
    "    success,frame1 = vidcap.read()\n",
    "    frame1 = cv.cvtColor(frame1, cv.COLOR_BGR2RGB)\n",
    "    frame1 = Image.fromarray(frame1).crop((0, 170, 640, 370)).resize((160,50))\n",
    "    while success:\n",
    "        if (count % 100 == 0) and count > 0:\n",
    "            print(count)\n",
    "        success,frame2 = vidcap.read()\n",
    "        if success == True:\n",
    "            frame2 = cv.cvtColor(frame2, cv.COLOR_BGR2RGB)\n",
    "            frame2 = Image.fromarray(frame2).crop((0, 170, 640, 370)).resize((160,50))\n",
    "            if augment_frames == True:\n",
    "                if random.random() > .85:\n",
    "                    frame1, frame2 = augment(frame1, frame2)\n",
    "            flow = opticalFlowDense(frame1, frame2)\n",
    "            op_flows.append(flow)\n",
    "            frames.append(np.array(frame2))\n",
    "            frame1 = frame2\n",
    "            count+=1\n",
    "        else:\n",
    "            print(\"video reading completed\")\n",
    "            continue\n",
    "    return np.array(frames), np.array(op_flows), count\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "hdf5_path = './train_data/train.hdf5'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# '''Only use to reset dataset'''\n",
    "# with h5py.File(hdf5_path, \"w\") as f:\n",
    "#     f.create_dataset(\"frame\", shape = (1, resize_frame_size[0], resize_frame_size[1], resize_frame_size[2]),\n",
    "#                     maxshape = (None, resize_frame_size[0], resize_frame_size[1], resize_frame_size[2]), \n",
    "#                     chunks = (1, resize_frame_size[0], resize_frame_size[1], resize_frame_size[2]))\n",
    "#     f.create_dataset(\"op_flow\", shape = (1, resize_size[0], resize_size[1], resize_size[2]),\n",
    "#                     maxshape = (None, resize_size[0], resize_size[1], resize_size[2]), \n",
    "#                     chunks = (1, resize_size[0], resize_size[1], resize_size[2]))\n",
    "#     f.create_dataset(\"speed\", shape = (1,), maxshape = (None,))\n",
    "# f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_hdf5(hdf5_path, frames, op_flows, resampled_speeds):\n",
    "    with h5py.File(hdf5_path) as f:\n",
    "        print(len(frames), len(op_flows), len(resampled_speeds))\n",
    "        print(f[\"frame\"], f[\"op_flow\"], f[\"speed\"])\n",
    "        f[\"frame\"].resize(f[\"frame\"].len() + len(frames), axis = 0)\n",
    "        f[\"op_flow\"].resize(f[\"op_flow\"].len() + len(op_flows), axis = 0)\n",
    "        f[\"speed\"].resize(f[\"speed\"].len() + len(resampled_speeds), axis = 0)\n",
    "        f[\"frame\"][-len(frames):] = frames\n",
    "        f[\"op_flow\"][-len(op_flows):] = op_flows\n",
    "        f[\"speed\"][-len(resampled_speeds):] = resampled_speeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def archive_processed(video_dir):\n",
    "    processed_dir = video_dir.split(\"/\")\n",
    "    processed_dir[-2] = \"processed\"\n",
    "    processed_dir = \"/\".join(processed_dir)\n",
    "    shutil.move(video_dir, processed_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (103787, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (103787, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (103787,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (104788, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (104788, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (104788,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (105789, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (105789, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (105789,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (106790, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (106790, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (106790,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (107791, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (107791, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (107791,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (108792, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (108792, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (108792,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (109793, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (109793, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (109793,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (110794, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (110794, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (110794,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (111795, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (111795, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (111795,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (112796, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (112796, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (112796,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (113797, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (113797, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (113797,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (114798, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (114798, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (114798,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (115799, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (115799, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (115799,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (116800, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (116800, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (116800,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (117801, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (117801, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (117801,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (118802, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (118802, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (118802,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (119803, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (119803, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (119803,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (120804, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (120804, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (120804,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (121805, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (121805, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (121805,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (122806, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (122806, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (122806,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (123807, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (123807, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (123807,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (124808, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (124808, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (124808,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (125809, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (125809, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (125809,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (126810, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (126810, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (126810,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (127811, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (127811, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (127811,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (128812, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (128812, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (128812,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (129813, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (129813, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (129813,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (130814, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (130814, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (130814,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (131815, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (131815, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (131815,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (132816, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (132816, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (132816,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (133817, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (133817, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (133817,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (134818, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (134818, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (134818,), type \"<f4\">\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (135819, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (135819, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (135819,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (136820, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (136820, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (136820,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (137821, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (137821, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (137821,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (138822, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (138822, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (138822,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (139823, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (139823, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (139823,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (140824, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (140824, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (140824,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (141825, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (141825, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (141825,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1000 1000 1000\n",
      "<HDF5 dataset \"frame\": shape (142826, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (142826, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (142826,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (143826, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (143826, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (143826,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (144827, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (144827, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (144827,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1000 1000 1000\n",
      "<HDF5 dataset \"frame\": shape (145828, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (145828, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (145828,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (146828, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (146828, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (146828,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (147829, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (147829, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (147829,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (148830, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (148830, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (148830,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (149831, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (149831, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (149831,), type \"<f4\">\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "video reading completed\n",
      "1001 1001 1001\n",
      "<HDF5 dataset \"frame\": shape (150832, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (150832, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (150832,), type \"<f4\">\n"
     ]
    }
   ],
   "source": [
    "#preprocess custom data\n",
    "for video in raw_videos:\n",
    "    video_dir = preprocessing_dir + \"/\" + video\n",
    "    if video in clipped_videos:\n",
    "        files = os.listdir(video_dir)\n",
    "        if \"fcamera.hevc\" in files and \"rlog.bz2\" in files:\n",
    "            downscaled_vid = convert_video(downscaled_dir, video_dir)\n",
    "            frames, op_flows, count = op_flow_video(downscaled_vid)\n",
    "            resampled_speeds = extract_logs(video_dir, frames.shape[0])\n",
    "            write_hdf5(hdf5_path, frames, op_flows, resampled_speeds)\n",
    "    archive_processed(video_dir)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open(\"./train_data/train.txt\", \"r\") as f:\n",
    "#     speeds = f.read()\n",
    "# speeds = [float(i) for i in speeds.split(\"\\n\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "1100\n",
      "1200\n",
      "1300\n",
      "1400\n",
      "1500\n",
      "1600\n",
      "1700\n",
      "1800\n",
      "1900\n",
      "2000\n",
      "2100\n",
      "2200\n",
      "2300\n",
      "2400\n",
      "2500\n",
      "2600\n",
      "2700\n",
      "2800\n",
      "2900\n",
      "3000\n",
      "3100\n",
      "3200\n",
      "3300\n",
      "3400\n",
      "3500\n",
      "3600\n",
      "3700\n",
      "3800\n",
      "3900\n",
      "4000\n",
      "4100\n",
      "4200\n",
      "4300\n",
      "4400\n",
      "4500\n",
      "4600\n",
      "4700\n",
      "4800\n",
      "4900\n",
      "5000\n",
      "5100\n",
      "5200\n",
      "5300\n",
      "5400\n",
      "5500\n",
      "5600\n",
      "5700\n",
      "5800\n",
      "5900\n",
      "6000\n",
      "6100\n",
      "6200\n",
      "6300\n",
      "6400\n",
      "6500\n",
      "6600\n",
      "6700\n",
      "6800\n",
      "6900\n",
      "7000\n",
      "7100\n",
      "7200\n",
      "7300\n",
      "7400\n",
      "7500\n",
      "7600\n",
      "7700\n",
      "7800\n",
      "7900\n",
      "8000\n",
      "8100\n",
      "8200\n",
      "8300\n",
      "8400\n",
      "8500\n",
      "8600\n",
      "8700\n",
      "8800\n",
      "8900\n",
      "9000\n",
      "9100\n",
      "9200\n",
      "9300\n",
      "9400\n",
      "9500\n",
      "9600\n",
      "9700\n",
      "9800\n",
      "9900\n",
      "10000\n",
      "10100\n",
      "10200\n",
      "10300\n",
      "10400\n",
      "10500\n",
      "10600\n",
      "10700\n",
      "10800\n",
      "10900\n",
      "11000\n",
      "11100\n",
      "11200\n",
      "11300\n",
      "11400\n",
      "11500\n",
      "11600\n",
      "11700\n",
      "11800\n",
      "11900\n",
      "12000\n",
      "12100\n",
      "12200\n",
      "12300\n",
      "12400\n",
      "12500\n",
      "12600\n",
      "12700\n",
      "12800\n",
      "12900\n",
      "13000\n",
      "13100\n",
      "13200\n",
      "13300\n",
      "13400\n",
      "13500\n",
      "13600\n",
      "13700\n",
      "13800\n",
      "13900\n",
      "14000\n",
      "14100\n",
      "14200\n",
      "14300\n",
      "14400\n",
      "14500\n",
      "14600\n",
      "14700\n",
      "14800\n",
      "14900\n",
      "15000\n",
      "15100\n",
      "15200\n",
      "15300\n",
      "15400\n",
      "15500\n",
      "15600\n",
      "15700\n",
      "15800\n",
      "15900\n",
      "16000\n",
      "16100\n",
      "16200\n",
      "16300\n",
      "16400\n",
      "16500\n",
      "16600\n",
      "16700\n",
      "16800\n",
      "16900\n",
      "17000\n",
      "17100\n",
      "17200\n",
      "17300\n",
      "17400\n",
      "17500\n",
      "17600\n",
      "17700\n",
      "17800\n",
      "17900\n",
      "18000\n",
      "18100\n",
      "18200\n",
      "18300\n",
      "18400\n",
      "18500\n",
      "18600\n",
      "18700\n",
      "18800\n",
      "18900\n",
      "19000\n",
      "19100\n",
      "19200\n",
      "19300\n",
      "19400\n",
      "19500\n",
      "19600\n",
      "19700\n",
      "19800\n",
      "19900\n",
      "20000\n",
      "20100\n",
      "20200\n",
      "20300\n",
      "video reading completed\n",
      "20399 20399 20400\n",
      "<HDF5 dataset \"frame\": shape (1, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (1, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (1,), type \"<f4\">\n"
     ]
    }
   ],
   "source": [
    "# #preprocess comma train data\n",
    "# comma_train_vid = './train_data/train.mp4'\n",
    "# hdf5_path_comma_train = './train_data/comma_train.hdf5'\n",
    "\n",
    "# '''Only use to reset dataset'''\n",
    "# with h5py.File(hdf5_path_comma_train, \"w\") as f:\n",
    "#     f.create_dataset(\"frame\", shape = (1, resize_frame_size[0], resize_frame_size[1], resize_frame_size[2]),\n",
    "#                     maxshape = (None, resize_frame_size[0], resize_frame_size[1], resize_frame_size[2]), \n",
    "#                     chunks = (1, resize_frame_size[0], resize_frame_size[1], resize_frame_size[2]))\n",
    "#     f.create_dataset(\"op_flow\", shape = (1, resize_size[0], resize_size[1], resize_size[2]),\n",
    "#                     maxshape = (None, resize_size[0], resize_size[1], resize_size[2]), \n",
    "#                     chunks = (1, resize_size[0], resize_size[1], resize_size[2]))\n",
    "#     f.create_dataset(\"speed\", shape = (1,), maxshape = (None,))\n",
    "# f.close()\n",
    "\n",
    "# frames, op_flows, count = op_flow_video(comma_train_vid, augment_frames = False)\n",
    "# write_hdf5(hdf5_path_comma_train, frames, op_flows, speeds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "1100\n",
      "1200\n",
      "1300\n",
      "1400\n",
      "1500\n",
      "1600\n",
      "1700\n",
      "1800\n",
      "1900\n",
      "2000\n",
      "2100\n",
      "2200\n",
      "2300\n",
      "2400\n",
      "2500\n",
      "2600\n",
      "2700\n",
      "2800\n",
      "2900\n",
      "3000\n",
      "3100\n",
      "3200\n",
      "3300\n",
      "3400\n",
      "3500\n",
      "3600\n",
      "3700\n",
      "3800\n",
      "3900\n",
      "4000\n",
      "4100\n",
      "4200\n",
      "4300\n",
      "4400\n",
      "4500\n",
      "4600\n",
      "4700\n",
      "4800\n",
      "4900\n",
      "5000\n",
      "5100\n",
      "5200\n",
      "5300\n",
      "5400\n",
      "5500\n",
      "5600\n",
      "5700\n",
      "5800\n",
      "5900\n",
      "6000\n",
      "6100\n",
      "6200\n",
      "6300\n",
      "6400\n",
      "6500\n",
      "6600\n",
      "6700\n",
      "6800\n",
      "6900\n",
      "7000\n",
      "7100\n",
      "7200\n",
      "7300\n",
      "7400\n",
      "7500\n",
      "7600\n",
      "7700\n",
      "7800\n",
      "7900\n",
      "8000\n",
      "8100\n",
      "8200\n",
      "8300\n",
      "8400\n",
      "8500\n",
      "8600\n",
      "8700\n",
      "8800\n",
      "8900\n",
      "9000\n",
      "9100\n",
      "9200\n",
      "9300\n",
      "9400\n",
      "9500\n",
      "9600\n",
      "9700\n",
      "9800\n",
      "9900\n",
      "10000\n",
      "10100\n",
      "10200\n",
      "10300\n",
      "10400\n",
      "10500\n",
      "10600\n",
      "10700\n",
      "video reading completed\n",
      "10797 10797 10797\n",
      "<HDF5 dataset \"frame\": shape (1, 50, 160, 3), type \"<f4\"> <HDF5 dataset \"op_flow\": shape (1, 50, 160, 2), type \"<f4\"> <HDF5 dataset \"speed\": shape (1,), type \"<f4\">\n"
     ]
    }
   ],
   "source": [
    "# #preprocess comma data\n",
    "# comma_test_vid = './test_data/test.mp4'\n",
    "# hdf5_path_comma_test = './test_data/comma_test.hdf5'\n",
    "\n",
    "# '''Only use to reset dataset'''\n",
    "# with h5py.File(hdf5_path_comma_test, \"w\") as f:\n",
    "#     f.create_dataset(\"frame\", shape = (1, resize_frame_size[0], resize_frame_size[1], resize_frame_size[2]),\n",
    "#                     maxshape = (None, resize_frame_size[0], resize_frame_size[1], resize_frame_size[2]), \n",
    "#                     chunks = (1, resize_frame_size[0], resize_frame_size[1], resize_frame_size[2]))\n",
    "#     f.create_dataset(\"op_flow\", shape = (1, resize_size[0], resize_size[1], resize_size[2]),\n",
    "#                     maxshape = (None, resize_size[0], resize_size[1], resize_size[2]), \n",
    "#                     chunks = (1, resize_size[0], resize_size[1], resize_size[2]))\n",
    "#     f.create_dataset(\"speed\", shape = (1,), maxshape = (None,))\n",
    "# f.close()\n",
    "\n",
    "# frames, op_flows, count = op_flow_video(comma_test_vid, augment_frames = False)\n",
    "# null_speeds = np.zeros(count)\n",
    "# write_hdf5(hdf5_path_comma_test, frames, op_flows, null_speeds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def extract_video(video_dir):\n",
    "#     extracting = True\n",
    "#     count = 0\n",
    "#     skipped_frames = 0\n",
    "#     while extracting == True:\n",
    "#         if count % 5 == 0:\n",
    "#             count +=1\n",
    "#             skipped_frames += 1\n",
    "#         try:\n",
    "#             fr = FrameReader(video_dir + '/fcamera.hevc')\n",
    "#             fr.get(count, pix_fmt='rgb24')[0]\n",
    "#             count += 1\n",
    "#         except:\n",
    "#             extracting = False\n",
    "#     return count - skipped_frames\n",
    "# extract_video(video_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}