data_split.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import pandas as pd\n",
    "import os\n",
    "from sklearn.model_selection import train_test_split\n",
    "import shutil"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bethalm Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "files_ocr_bethalm = sorted([f for f in os.listdir(\"OCR/final/Bentham\") if f.endswith('.txt')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ocr_002_112_001.txt'"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "files_ocr_bethalm[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'GT_002_112_001.txt'"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "files_gt_bethalm = sorted([f for f in os.listdir(\"data/BenthamDataset/GT/GT_Extracted\") if f.endswith('.txt')])\n",
    "files_gt_bethalm[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# len(files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# list(range(len(files)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "train, test = train_test_split(list(range(len(files_ocr_bethalm))), test_size=0.3, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "val, test = train_test_split(test,test_size=1/3,random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in train: \n",
    "    source_ocr = \"/Users/amandanitta/Desktop/GitHub/ICS661/digitalize_handwritten/OCR/final/Bentham/\" + files_ocr_bethalm[i]\n",
    "    destination_ocr = \"dataForModel/train/Betham/OCR/\" + files_ocr_bethalm[i]\n",
    "    shutil.copy(source_ocr,destination_ocr)\n",
    "\n",
    "    source_gt = \"data/BenthamDataset/GT/GT_Extracted/\" + files_gt_bethalm[i]\n",
    "    destination_gt = \"dataForModel/train/Betham/GT/\" + files_gt_bethalm[i]\n",
    "    shutil.copy(source_gt,destination_gt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in val: \n",
    "    source_ocr = \"/Users/amandanitta/Desktop/GitHub/ICS661/digitalize_handwritten/OCR/final/Bentham/\" + files_ocr_bethalm[i]\n",
    "    destination_ocr = \"dataForModel/validation/Betham/OCR/\" + files_ocr_bethalm[i]\n",
    "    shutil.copy(source_ocr,destination_ocr)\n",
    "\n",
    "    source_gt = \"data/BenthamDataset/GT/GT_Extracted/\" + files_gt_bethalm[i]\n",
    "    destination_gt = \"dataForModel/validation/Betham/GT/\" + files_gt_bethalm[i]\n",
    "    shutil.copy(source_gt,destination_gt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in test: \n",
    "    source_ocr = \"/Users/amandanitta/Desktop/GitHub/ICS661/digitalize_handwritten/OCR/final/Bentham/\" + files_ocr_bethalm[i]\n",
    "    destination_ocr = \"dataForModel/test/Betham/OCR/\" + files_ocr_bethalm[i]\n",
    "    shutil.copy(source_ocr,destination_ocr)\n",
    "\n",
    "    source_gt = \"data/BenthamDataset/GT/GT_Extracted/\" + files_gt_bethalm[i]\n",
    "    destination_gt = \"dataForModel/test/Betham/GT/\" + files_gt_bethalm[i]\n",
    "    shutil.copy(source_gt,destination_gt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IAM Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "files_ocr_IAM_AD = sorted([f for f in os.listdir(\"OCR/final/IAM/formsAD/\") if f.endswith('.txt')])\n",
    "files_ocr_IAM_EH = sorted([f for f in os.listdir(\"OCR/final/IAM/formsEH/\") if f.endswith('.txt')])\n",
    "files_ocr_IAM_IZ = sorted([f for f in os.listdir(\"OCR/final/IAM/formsiZ/\") if f.endswith('.txt')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'GT_a01-000u.txt'"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "files_gt_IAM = sorted([f for f in os.listdir(\"data/IAM_database/GT/GT_extract\") if f.endswith('.txt')])\n",
    "files_gt_IAM[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "files_IAM_AD, files_IAM_EH, files_IAM_IZ = len(files_ocr_IAM_AD), len(files_ocr_IAM_EH), len(files_ocr_IAM_IZ)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_AD, test_AD = train_test_split(list(range(files_IAM_AD)), test_size=0.3, random_state=42)\n",
    "val_AD, test_AD = train_test_split(test,test_size=1/3,random_state=42)\n",
    "\n",
    "IAM_gt_AD_train = [\"GT_\"+files_ocr_IAM_AD[i][4:] for i in train_AD]\n",
    "IAM_gt_AD_val = [\"GT_\"+files_ocr_IAM_AD[i][4:] for i in val_AD]\n",
    "IAM_gt_AD_test = [\"GT_\"+files_ocr_IAM_AD[i][4:] for i in test_AD]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_EH, test_EH = train_test_split(list(range(files_IAM_EH)), test_size=0.3, random_state=42)\n",
    "val_EH, test_EH = train_test_split(test,test_size=1/3,random_state=42)\n",
    "\n",
    "IAM_gt_EH_train = [\"GT_\"+files_ocr_IAM_EH[i][4:] for i in train_EH]\n",
    "IAM_gt_EH_val = [\"GT_\"+files_ocr_IAM_EH[i][4:] for i in val_EH]\n",
    "IAM_gt_EH_test = [\"GT_\"+files_ocr_IAM_EH[i][4:] for i in test_EH]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_IZ, test_IZ = train_test_split(list(range(files_IAM_IZ)), test_size=0.3, random_state=42)\n",
    "val_IZ, test_IZ = train_test_split(test,test_size=1/3,random_state=42)\n",
    "\n",
    "IAM_gt_IZ_train = [\"GT_\"+files_ocr_IAM_IZ[i][4:] for i in train_IZ]\n",
    "IAM_gt_IZ_val = [\"GT_\"+files_ocr_IAM_IZ[i][4:] for i in val_IZ]\n",
    "IAM_gt_IZ_test = [\"GT_\"+files_ocr_IAM_IZ[i][4:] for i in test_IZ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "IAM_gt_train = IAM_gt_AD_train + IAM_gt_EH_train + IAM_gt_IZ_train\n",
    "IAM_gt_val = IAM_gt_AD_val + IAM_gt_EH_val + IAM_gt_IZ_val\n",
    "IAM_gt_test = IAM_gt_AD_test + IAM_gt_EH_test + IAM_gt_IZ_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GT IAM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in IAM_gt_train: \n",
    "    source_gt = \"data/IAM_database/GT/GT_extract/\" + i\n",
    "    destination_gt = \"dataForModel/train/IAM/GT/\" + i\n",
    "    shutil.copy(source_gt,destination_gt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in IAM_gt_val: \n",
    "    source_gt = \"data/IAM_database/GT/GT_extract/\" + i\n",
    "    destination_gt = \"dataForModel/validation/IAM/GT/\" + i\n",
    "    shutil.copy(source_gt,destination_gt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in IAM_gt_test: \n",
    "    source_gt = \"data/IAM_database/GT/GT_extract/\" + i\n",
    "    destination_gt = \"dataForModel/test/IAM/GT/\" + i\n",
    "    shutil.copy(source_gt,destination_gt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## OCR IAM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training IAM OCR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in train_AD: \n",
    "    source_ocr = \"OCR/final/IAM/formsAD/\" + files_ocr_IAM_AD[i]\n",
    "    destination_ocr = \"dataForModel/train/IAM/OCR/\" + files_ocr_IAM_AD[i]\n",
    "    shutil.copy(source_ocr,destination_ocr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in train_EH:\n",
    "    source_ocr = \"OCR/final/IAM/formsEH/\" + files_ocr_IAM_EH[i]\n",
    "    destination_ocr = \"dataForModel/train/IAM/OCR/\" + files_ocr_IAM_EH[i]\n",
    "    shutil.copy(source_ocr,destination_ocr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in train_IZ: \n",
    "    source_ocr = \"OCR/final/IAM/formsIZ/\" + files_ocr_IAM_IZ[i]\n",
    "    destination_ocr = \"dataForModel/train/IAM/OCR/\" + files_ocr_IAM_IZ[i]\n",
    "    shutil.copy(source_ocr,destination_ocr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Validation IAM OCR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in val_AD: \n",
    "    source_ocr = \"OCR/final/IAM/formsAD/\" + files_ocr_IAM_AD[i]\n",
    "    destination_ocr = \"dataForModel/validation/IAM/OCR/\" + files_ocr_IAM_AD[i]\n",
    "    shutil.copy(source_ocr,destination_ocr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in val_EH:\n",
    "    source_ocr = \"OCR/final/IAM/formsEH/\" + files_ocr_IAM_EH[i]\n",
    "    destination_ocr = \"dataForModel/validation/IAM/OCR/\" + files_ocr_IAM_EH[i]\n",
    "    shutil.copy(source_ocr,destination_ocr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in val_IZ: \n",
    "    source_ocr = \"OCR/final/IAM/formsIZ/\" + files_ocr_IAM_IZ[i]\n",
    "    destination_ocr = \"dataForModel/validation/IAM/OCR/\" + files_ocr_IAM_IZ[i]\n",
    "    shutil.copy(source_ocr,destination_ocr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test IAM OCR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in test_AD: \n",
    "    source_ocr = \"OCR/final/IAM/formsAD/\" + files_ocr_IAM_AD[i]\n",
    "    destination_ocr = \"dataForModel/test/IAM/OCR/\" + files_ocr_IAM_AD[i]\n",
    "    shutil.copy(source_ocr,destination_ocr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in test_EH:\n",
    "    source_ocr = \"OCR/final/IAM/formsEH/\" + files_ocr_IAM_EH[i]\n",
    "    destination_ocr = \"dataForModel/test/IAM/OCR/\" + files_ocr_IAM_EH[i]\n",
    "    shutil.copy(source_ocr,destination_ocr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in test_IZ: \n",
    "    source_ocr = \"OCR/final/IAM/formsIZ/\" + files_ocr_IAM_IZ[i]\n",
    "    destination_ocr = \"dataForModel/test/IAM/OCR/\" + files_ocr_IAM_IZ[i]\n",
    "    shutil.copy(source_ocr,destination_ocr)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}