Tf_main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\user\\Miniconda3\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "C:\\Users\\user\\Miniconda3\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "C:\\Users\\user\\Miniconda3\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:528: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "C:\\Users\\user\\Miniconda3\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:529: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "C:\\Users\\user\\Miniconda3\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:530: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "C:\\Users\\user\\Miniconda3\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:535: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "from typing import Tuple\n",
    "import io\n",
    "import time\n",
    "import cv2\n",
    "import torch\n",
    "import torchvision\n",
    "from PIL import Image, ImageOps\n",
    "from my_mobilenet_v2_tsm import MobileNetV2\n",
    "from onnx_tf.backend import prepare\n",
    "import onnx\n",
    "import tensorflow as tf\n",
    "from onnx import version_converter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mobilenet_v2_tsm2 import MobileNetV2\n",
    "\n",
    "from my_mobilenet_v2_tsm import MobileNetV2 as MobilenetTsmLstm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "SOFTMAX_THRES = 0\n",
    "HISTORY_LOGIT = True\n",
    "REFINE_OUTPUT = True\n",
    "\n",
    "# scale\n",
    "\n",
    "\n",
    "def get_executor(use_gpu=True):\n",
    "    torch_inputs = (torch.rand([1, 3, 224, 224]),\n",
    "                    torch.zeros([1, 3, 56, 56]),\n",
    "                    torch.zeros([1, 4, 28, 28]),\n",
    "                    torch.zeros([1, 4, 28, 28]),\n",
    "                    torch.zeros([1, 8, 14, 14]),\n",
    "                    torch.zeros([1, 8, 14, 14]),\n",
    "                    torch.zeros([1, 8, 14, 14]),\n",
    "                    torch.zeros([1, 12, 14, 14]),\n",
    "                    torch.zeros([1, 12, 14, 14]),\n",
    "                    torch.zeros([1, 20, 7, 7]),\n",
    "                    torch.zeros([1, 20, 7, 7]))\n",
    "    torch_module = MobileNetV2(n_class=27)\n",
    "    input_names = []\n",
    "    input_shapes = {}\n",
    "    torch_module.eval()\n",
    "    if not os.path.exists(\"mobilenetv2_jester_online.pth.tar\"):  # checkpoint not downloaded\n",
    "        print('Downloading PyTorch checkpoint...')\n",
    "        import urllib.request\n",
    "        url = 'https://hanlab.mit.edu/projects/tsm/models/mobilenetv2_jester_online.pth.tar'\n",
    "        urllib.request.urlretrieve(url, './mobilenetv2_jester_online.pth.tar')\n",
    "    #print(torch.load(\"mobilenetv2_jester_online.pth.tar\"))\n",
    "    torch_module.load_state_dict(torch.load(\"mobilenetv2_jester_online.pth.tar\"), strict=False)\n",
    "    with torch.no_grad():\n",
    "        for index, torch_input in enumerate(torch_inputs):\n",
    "            name = \"i\" + str(index)\n",
    "            input_names.append(name)\n",
    "            input_shapes[name] = torch_input.shape\n",
    "        buffer = io.BytesIO()\n",
    "        torch.onnx.export(torch_module, torch_inputs, buffer, input_names=input_names, output_names=[\"o\" + str(i) for i in range(len(torch_inputs))])\n",
    "\n",
    "        outs = torch_module(*torch_inputs)\n",
    "        buffer.seek(0, 0)\n",
    "        onnx_model = onnx.load(buffer)\n",
    "        #onnx.checker.check_model(onnx_model)\n",
    "        #onnx_model = onnx.load(\"BERT.onnx\")  # load onnx model\n",
    "        tf_rep = prepare(onnx_model)  # prepare tf representation\n",
    "    return tf_rep \n",
    "\n",
    "\n",
    "def transform(frame: np.ndarray):\n",
    "    # 480, 640, 3, 0 ~ 255\n",
    "    frame = cv2.resize(frame, (224, 224))  # (224, 224, 3) 0 ~ 255\n",
    "    frame = frame / 255.0  # (224, 224, 3) 0 ~ 1.0\n",
    "    frame = np.transpose(frame, axes=[2, 0, 1])  # (3, 224, 224) 0 ~ 1.0\n",
    "    frame = np.expand_dims(frame, axis=0)  # (1, 3, 480, 640) 0 ~ 1.0\n",
    "    return frame\n",
    "\n",
    "\n",
    "class GroupScale(object):\n",
    "    \"\"\" Rescales the input PIL.Image to the given 'size'.\n",
    "    'size' will be the size of the smaller edge.\n",
    "    For example, if height > width, then image will be\n",
    "    rescaled to (size * height / width, size)\n",
    "    size: size of the smaller edge\n",
    "    interpolation: Default: PIL.Image.BILINEAR\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, size, interpolation=Image.BILINEAR):\n",
    "        self.worker = torchvision.transforms.Resize(size, interpolation)\n",
    "\n",
    "    def __call__(self, img_group):\n",
    "        return [self.worker(img) for img in img_group]\n",
    "\n",
    "\n",
    "class GroupCenterCrop(object):\n",
    "    def __init__(self, size):\n",
    "        self.worker = torchvision.transforms.CenterCrop(size)\n",
    "\n",
    "    def __call__(self, img_group):\n",
    "        return [self.worker(img) for img in img_group]\n",
    "\n",
    "\n",
    "class Stack(object):\n",
    "\n",
    "    def __init__(self, roll=False):\n",
    "        self.roll = roll\n",
    "\n",
    "    def __call__(self, img_group):\n",
    "        if img_group[0].mode == 'L':\n",
    "            return np.concatenate([np.expand_dims(x, 2) for x in img_group], axis=2)\n",
    "        elif img_group[0].mode == 'RGB':\n",
    "            if self.roll:\n",
    "                return np.concatenate([np.array(x)[:, :, ::-1] for x in img_group], axis=2)\n",
    "            else:\n",
    "                return np.concatenate(img_group, axis=2)\n",
    "\n",
    "\n",
    "class ToTorchFormatTensor(object):\n",
    "    \"\"\" Converts a PIL.Image (RGB) or numpy.ndarray (H x W x C) in the range [0, 255]\n",
    "    to a torch.FloatTensor of shape (C x H x W) in the range [0.0, 1.0] \"\"\"\n",
    "\n",
    "    def __init__(self, div=True):\n",
    "        self.div = div\n",
    "\n",
    "    def __call__(self, pic):\n",
    "        if isinstance(pic, np.ndarray):\n",
    "            # handle numpy array\n",
    "            \n",
    "            img = torch.from_numpy(pic).permute(2, 0, 1).contiguous()\n",
    "        else:\n",
    "            # handle PIL Image\n",
    "            img = torch.ByteTensor(torch.ByteStorage.from_buffer(pic.tobytes()))\n",
    "            img = img.view(pic.size[1], pic.size[0], len(pic.mode))\n",
    "            # put it from HWC to CHW format\n",
    "            # yikes, this transpose takes 80% of the loading time/CPU\n",
    "            img = img.transpose(0, 1).transpose(0, 2).contiguous()\n",
    "        return img.float().div(255) if self.div else img.float()\n",
    "\n",
    "\n",
    "class GroupNormalize(object):\n",
    "    def __init__(self, mean, std):\n",
    "        self.mean = mean\n",
    "        self.std = std\n",
    "\n",
    "    def __call__(self, tensor):\n",
    "        rep_mean = self.mean * (tensor.size()[0] // len(self.mean))\n",
    "        rep_std = self.std * (tensor.size()[0] // len(self.std))\n",
    "\n",
    "        # TODO: make efficient\n",
    "        for t, m, s in zip(tensor, rep_mean, rep_std):\n",
    "            t.sub_(m).div_(s)\n",
    "\n",
    "        return tensor\n",
    "\n",
    "\n",
    "def get_transform():\n",
    "    cropping = torchvision.transforms.Compose([\n",
    "        GroupScale(256),\n",
    "        GroupCenterCrop(224),\n",
    "    ])\n",
    "    transform = torchvision.transforms.Compose([\n",
    "        cropping,\n",
    "        Stack(roll=False),\n",
    "        ToTorchFormatTensor(div=True),\n",
    "        GroupNormalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ])\n",
    "    return transform\n",
    "\n",
    "catigories = [\n",
    "    \"Doing other things\",  # 0\n",
    "    \"Drumming Fingers\",  # 1\n",
    "    \"No gesture\",  # 2\n",
    "    \"Pulling Hand In\",  # 3\n",
    "    \"Pulling Two Fingers In\",  # 4\n",
    "    \"Pushing Hand Away\",  # 5\n",
    "    \"Pushing Two Fingers Away\",  # 6\n",
    "    \"Rolling Hand Backward\",  # 7\n",
    "    \"Rolling Hand Forward\",  # 8\n",
    "    \"Shaking Hand\",  # 9\n",
    "    \"Sliding Two Fingers Down\",  # 10\n",
    "    \"Sliding Two Fingers Left\",  # 11\n",
    "    \"Sliding Two Fingers Right\",  # 12\n",
    "    \"Sliding Two Fingers Up\",  # 13\n",
    "    \"Stop Sign\",  # 14\n",
    "    \"Swiping Down\",  # 15\n",
    "    \"Swiping Left\",  # 16\n",
    "    \"Swiping Right\",  # 17\n",
    "    \"Swiping Up\",  # 18\n",
    "    \"Thumb Down\",  # 19\n",
    "    \"Thumb Up\",  # 20\n",
    "    \"Turning Hand Clockwise\",  # 21\n",
    "    \"Turning Hand Counterclockwise\",  # 22\n",
    "    \"Zooming In With Full Hand\",  # 23\n",
    "    \"Zooming In With Two Fingers\",  # 24\n",
    "    \"Zooming Out With Full Hand\",  # 25\n",
    "    \"Zooming Out With Two Fingers\"  # 26\n",
    "]\n",
    "\n",
    "\n",
    "n_still_frame = 0\n",
    "\n",
    "def process_output(idx_, history):\n",
    "    # idx_: the output of current frame\n",
    "    # history: a list containing the history of predictions\n",
    "    if not REFINE_OUTPUT:\n",
    "        return idx_, history\n",
    "\n",
    "    max_hist_len = 20  # max history buffer\n",
    "\n",
    "    # mask out illegal action\n",
    "    if idx_ in [7, 8, 21, 22, 3]:\n",
    "        idx_ = history[-1]\n",
    "\n",
    "    # use only single no action class\n",
    "    if idx_ == 0:\n",
    "        idx_ = 2\n",
    "    \n",
    "    # history smoothing\n",
    "    if idx_ != history[-1]:\n",
    "        if not (history[-1] == history[-2]): #  and history[-2] == history[-3]):\n",
    "            idx_ = history[-1]\n",
    "    \n",
    "\n",
    "    history.append(idx_)\n",
    "    history = history[-max_hist_len:]\n",
    "\n",
    "    return history[-1], history\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Open camera...\n",
      "<class 'numpy.ndarray'> (480, 640, 3) <class 'cv2.VideoCapture'>\n",
      "Build transformer...\n",
      "Build Executor...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\chil\\mobilenet_v2_tsm2.py:109: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
      "  c0 = int(copy.deepcopy(c)) // 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ready!\n",
      "<class 'numpy.ndarray'> (480, 640, 3) <class 'cv2.VideoCapture'>\n",
      "torch.Size([1, 3, 224, 224])\n",
      "<class 'tensorflow.python.ops.variables.RefVariable'>\n",
      "(1, 3, 224, 224)\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "setting an array element with a sequence.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;31mTypeError\u001b[0m: float() argument must be a string or a number, not 'RefVariable'",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-5-9d6b8f76ff58>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m    139\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    140\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 141\u001b[1;33m \u001b[0mmain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-5-9d6b8f76ff58>\u001b[0m in \u001b[0;36mmain\u001b[1;34m()\u001b[0m\n\u001b[0;32m     76\u001b[0m             \u001b[1;31m#out = model(img_tran, *buffer)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     77\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 78\u001b[1;33m             \u001b[0mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mimg_tran\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     79\u001b[0m             \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     80\u001b[0m             \u001b[1;31m#print(torch.argmax(x))\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\onnx_tf\\backend_rep.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, inputs, **kwargs)\u001b[0m\n\u001b[0;32m     46\u001b[0m         ]\n\u001b[0;32m     47\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 48\u001b[1;33m         \u001b[0moutput_values\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msess\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mexternal_output\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     49\u001b[0m         return namedtupledict('Outputs',\n\u001b[0;32m     50\u001b[0m                               self.predict_net.external_output)(*output_values)\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    927\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    928\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 929\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    930\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    931\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1119\u001b[0m             \u001b[0mfeed_handles\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0msubfeed_t\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msubfeed_val\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1120\u001b[0m           \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1121\u001b[1;33m             \u001b[0mnp_val\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msubfeed_val\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msubfeed_dtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1122\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1123\u001b[0m           if (not is_tensor_handle_feed and\n",
      "\u001b[1;32m~\\Miniconda3\\lib\\site-packages\\numpy\\core\\_asarray.py\u001b[0m in \u001b[0;36masarray\u001b[1;34m(a, dtype, order)\u001b[0m\n\u001b[0;32m     83\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     84\u001b[0m     \"\"\"\n\u001b[1;32m---> 85\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     86\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     87\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: setting an array element with a sequence."
     ]
    }
   ],
   "source": [
    "\n",
    "WINDOW_NAME = 'Video Gesture Recognition'\n",
    "def main():\n",
    "    print(\"Open camera...\")\n",
    "    cap = cv2.VideoCapture(0)\n",
    "    _, img = cap.read()\n",
    "    print(type(img), img.shape,type(cap))\n",
    "    \n",
    "    \n",
    "    #print(cap)\n",
    "\n",
    "    # set a lower resolution for speed up\n",
    "    #cap.set(cv2.CAP_PROP_FRAME_WIDTH, 320)\n",
    "    #cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 240)\n",
    "    \n",
    "    # env variables\n",
    "    full_screen = False\n",
    "    cv2.namedWindow(WINDOW_NAME, cv2.WINDOW_NORMAL)\n",
    "    cv2.resizeWindow(WINDOW_NAME, 640, 480)\n",
    "    cv2.moveWindow(WINDOW_NAME, 0, 0)\n",
    "    cv2.setWindowTitle(WINDOW_NAME, WINDOW_NAME)\n",
    "\n",
    "\n",
    "    t = None\n",
    "    index = 0\n",
    "    print(\"Build transformer...\")\n",
    "    transform = get_transform()\n",
    "    print(\"Build Executor...\")\n",
    "    model = get_executor()\n",
    "    buffer = (\n",
    "        tf.Variable(np.empty((1, 3, 56, 56))),\n",
    "        tf.Variable(np.empty((1, 4, 28, 28))),\n",
    "        tf.Variable(np.empty((1, 4, 28, 28))),\n",
    "        tf.Variable(np.empty((1, 8, 14, 14))),\n",
    "        tf.Variable(np.empty((1, 8, 14, 14))),\n",
    "        tf.Variable(np.empty((1, 8, 14, 14))),\n",
    "        tf.Variable(np.empty((1, 12, 14, 14))),\n",
    "        tf.Variable(np.empty((1, 12, 14, 14))),\n",
    "        tf.Variable(np.empty((1, 20, 7, 7))),\n",
    "        tf.Variable(np.empty((1, 20, 7, 7)))\n",
    "    )\n",
    "    \"\"\"buffer = [\n",
    "            \n",
    "            torch.zeros([1, 3, 56, 56], dtype=torch.float32),\n",
    "            torch.zeros([1, 4, 28, 28], dtype=torch.float32),\n",
    "            torch.zeros([1, 4, 28, 28], dtype=torch.float32),\n",
    "            torch.zeros([1, 8, 14, 14], dtype=torch.float32),\n",
    "            torch.zeros([1, 8, 14, 14], dtype=torch.float32),\n",
    "            torch.zeros([1, 8, 14, 14], dtype=torch.float32),\n",
    "            torch.zeros([1, 12, 14, 14], dtype=torch.float32),\n",
    "            torch.zeros([1, 12, 14, 14], dtype=torch.float32),\n",
    "            torch.zeros([1, 20, 7, 7], dtype=torch.float32),\n",
    "            torch.zeros([1, 20, 7, 7], dtype=torch.float32)\n",
    "        ]\"\"\"\n",
    "    idx = 0\n",
    "    history = [2]\n",
    "    history_logit = []\n",
    "    history_timing = []\n",
    "\n",
    "    i_frame = -1\n",
    "\n",
    "    print(\"Ready!\")\n",
    "    while True:\n",
    "        i_frame += 1\n",
    "        _, img = cap.read()  # (480, 640, 3) 0 ~ 255\n",
    "        print(type(img), img.shape,type(cap))\n",
    "        if i_frame % 2 == 0:  # skip every other frame to obtain a suitable frame rate   \n",
    "            t1 = time.time()\n",
    "            #print(transform([Image.fromarray(img).convert('RGB')]).size())\n",
    "            img_tran = transform([Image.fromarray(img).convert('RGB')]).unsqueeze(0)\n",
    "            print(img_tran.shape)\n",
    "            img_tran = tf.Variable(img_tran)\n",
    "            print(type(img_tran))\n",
    "            print(img_tran.shape)\n",
    "            \n",
    "            #out = model(img_tran, *buffer)\n",
    "            \n",
    "            x = model.run((img_tran,) + buffer)\n",
    "            print(x)\n",
    "            #print(torch.argmax(x))\n",
    "            \n",
    "            idx_ = torch.argmax(x)\n",
    "            #idx_ = torch.max(x, 1)[1][0]\n",
    "            #print(idx_)\n",
    "            \n",
    "\n",
    "\n",
    "\n",
    "            idx, history = process_output(idx_, history)\n",
    "            #idx = idx_\n",
    "\n",
    "            t2 = time.time()\n",
    "            print(f\"{index} {catigories[idx]}\")\n",
    "\n",
    "\n",
    "            current_time = t2 - t1\n",
    "\n",
    "        img = cv2.resize(img, (640, 480))\n",
    "        img = img[:, ::-1]\n",
    "        height, width, _ = img.shape\n",
    "        label = np.zeros([height // 10, width, 3]).astype('uint8') + 255\n",
    "\n",
    "        cv2.putText(label, 'Prediction: ' + catigories[idx],\n",
    "                    (0, int(height / 16)),\n",
    "                    cv2.FONT_HERSHEY_SIMPLEX,\n",
    "                    0.7, (0, 0, 0), 2)\n",
    "        cv2.putText(label, '{:.1f} Vid/s'.format(1 / current_time),\n",
    "                    (width - 170, int(height / 16)),\n",
    "                    cv2.FONT_HERSHEY_SIMPLEX,\n",
    "                    0.7, (0, 0, 0), 2)\n",
    "\n",
    "        img = np.concatenate((img, label), axis=0)\n",
    "        cv2.imshow(WINDOW_NAME, img)\n",
    "\n",
    "        key = cv2.waitKey(1)\n",
    "        if key & 0xFF == ord('q') or key == 27:  # exit\n",
    "            break\n",
    "        elif key == ord('F') or key == ord('f'):  # full screen\n",
    "            print('Changing full screen option!')\n",
    "            full_screen = not full_screen\n",
    "            if full_screen:\n",
    "                print('Setting FS!!!')\n",
    "                cv2.setWindowProperty(WINDOW_NAME, cv2.WND_PROP_FULLSCREEN,\n",
    "                                      cv2.WINDOW_FULLSCREEN)\n",
    "            else:\n",
    "                cv2.setWindowProperty(WINDOW_NAME, cv2.WND_PROP_FULLSCREEN,\n",
    "                                      cv2.WINDOW_NORMAL)\n",
    "\n",
    "\n",
    "        if t is None:\n",
    "            t = time.time()\n",
    "        else:\n",
    "            nt = time.time()\n",
    "            index += 1\n",
    "            t = nt\n",
    "\n",
    "    cap.release()\n",
    "    cv2.destroyAllWindows()\n",
    "\n",
    "\n",
    "main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}