isot2hdf5.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# h5py document: http://docs.h5py.org/en/stable/quick.html\n",
    "import os\n",
    "import h5py\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from PIL import Image\n",
    "from tqdm.notebook import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _read_images(grayscale_path, image_name_list):\n",
    "    image_list = []\n",
    "    for name in tqdm(image_name_list, \\\n",
    "                                desc='Reading {0} images for conversion'.format(len(image_name_list))):\n",
    "        img = Image.open(grayscale_path + name).convert('L')\n",
    "        img = np.array(img).reshape(32, 32) / 255.0\n",
    "        image_list.append(img)\n",
    "    return image_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normal2hdf5(grayscale_path, csv_file_path, hdf5_file_path):\n",
    "    if os.path.exists(hdf5_file_path) == False:\n",
    "        os.mkdir(hdf5_file_path)\n",
    "        \n",
    "    name_label = pd.read_csv(csv_file_path, usecols=['PicName', 'Label'])\n",
    "\n",
    "    name_list = (np.array(name_label.loc[:, 'PicName']) + '.bmp').tolist()\n",
    "    label_list = np.array(name_label.loc[:, 'Label']).tolist()\n",
    "\n",
    "    image_num = len(name_list)\n",
    "    \n",
    "    assert image_num == len(label_list)\n",
    "    \n",
    "    print('Totaly {img_num} images of \\'normal\\' types.'\\\n",
    "         .format(img_num = image_num))\n",
    "    \n",
    "    # read images and save them as hdf5 each 50000 images\n",
    "    quotient = image_num // 50000 + 1\n",
    "    \n",
    "    for itr in range(1, quotient + 1): # start from 1 !!!!!!!!!!!!!!!!!!!!!!!!!!!\n",
    "        print('Converting {0}/{1} set of \\'normal\\' images'.format(itr, quotient))\n",
    "        if itr != quotient:\n",
    "            divided_name_list = name_list[(itr-1)*50000: itr*50000]\n",
    "            image_list = _read_images(grayscale_path, divided_name_list)\n",
    "        elif itr == quotient:\n",
    "            divided_name_list = name_list[(itr-1)*50000: image_num]\n",
    "            image_list = _read_images(grayscale_path, divided_name_list)\n",
    "        \n",
    "        with h5py.File('{path}normal_{cnt}.h5'.format(path=hdf5_file_path, cnt=itr)\\\n",
    "                       , 'w') as hdf:\n",
    "            hdf.create_dataset('image_array', data=np.array(image_list))\n",
    "    \n",
    "    print('{0} image sets saved as normal_number.h5 in {1}.'.format(quotient, grayscale_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Totaly 986708 images of 'normal' types.\n",
      "Converting 1/20 set of 'normal' images\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e16ced93883c4b8c8b0cbb85e2476aa8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Reading 50000 images for conversion', max=50000.0, style=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Converting 2/20 set of 'normal' images\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "07075872fcb84290a9646a8d1cda5864",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Reading 50000 images for conversion', max=50000.0, style=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Converting 3/20 set of 'normal' images\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b7f468fdb7d141c09f3d659cf0ec5541",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Reading 50000 images for conversion', max=50000.0, style=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Converting 4/20 set of 'normal' images\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f92a472159f54abe91d98380ba6be94f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Reading 50000 images for conversion', max=50000.0, style=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Converting 5/20 set of 'normal' images\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d2c51bcefb5544ba97ecb56499b57ace",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Reading 50000 images for conversion', max=50000.0, style=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Converting 6/20 set of 'normal' images\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5951b08c40974532b82bfd4cc50a367b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Reading 50000 images for conversion', max=50000.0, style=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Converting 7/20 set of 'normal' images\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "311423cef81747d3948d5222d5542141",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Reading 50000 images for conversion', max=50000.0, style=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Converting 8/20 set of 'normal' images\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1a25c224dc7e4f5dabcc2c72e5b75d34",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Reading 50000 images for conversion', max=50000.0, style=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Converting 9/20 set of 'normal' images\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "240c4411717a44f68af38bdbf4f39c52",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Reading 50000 images for conversion', max=50000.0, style=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Converting 10/20 set of 'normal' images\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ca71df9161d54a3d804fc50f59ea6116",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Reading 50000 images for conversion', max=50000.0, style=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Converting 11/20 set of 'normal' images\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e2eea2d86e314dd99822e650356975d7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Reading 50000 images for conversion', max=50000.0, style=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Converting 12/20 set of 'normal' images\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aed2dab2b5b34d5aa59eee3b9476509e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Reading 50000 images for conversion', max=50000.0, style=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Converting 13/20 set of 'normal' images\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aafdc00b18a24b70995fa847f6795b57",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Reading 50000 images for conversion', max=50000.0, style=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Converting 14/20 set of 'normal' images\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0842334053e4e3cac5b21694f45dfc4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Reading 50000 images for conversion', max=50000.0, style=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Converting 15/20 set of 'normal' images\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5445578f1656493fac649a2ef1d37e40",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Reading 50000 images for conversion', max=50000.0, style=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Converting 16/20 set of 'normal' images\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "445685fdac494efc854f7b643c92c022",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Reading 50000 images for conversion', max=50000.0, style=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Converting 17/20 set of 'normal' images\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d81252a56e0f400d8e317f601e2dac77",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Reading 50000 images for conversion', max=50000.0, style=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Converting 18/20 set of 'normal' images\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b08d11710bba4fe8bd26d5633f8e29a3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Reading 50000 images for conversion', max=50000.0, style=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Converting 19/20 set of 'normal' images\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02eac002d92f49538402a45e7fb19a56",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Reading 50000 images for conversion', max=50000.0, style=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Converting 20/20 set of 'normal' images\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7291e96c3954486cadfecc4e9e933646",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Reading 36708 images for conversion', max=36708.0, style=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "20 image sets saved as normal_number.h5 in /root/ISOT-2010/isot/grayscale/.\n"
     ]
    }
   ],
   "source": [
    "grayscale_path = '/root/ISOT-2010/isot/grayscale/'\n",
    "isot_csv = '/root/ISOT-2010/isot/isot.csv'\n",
    "hdf5_file_path = '/root/ISOT-2010/isot/normal/'\n",
    "\n",
    "normal2hdf5(grayscale_path, isot_csv, hdf5_file_path)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}