OnlineGame (Klooizooi).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "6a20f23f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e7daa81e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Checks if current position is not floating.\n",
    "def CheckBelow(Environment, X, Y, Z):\n",
    "    return Z == 0 or (Environment[X,Y,:Z] != '0').all()\n",
    "\n",
    "#Checks if current position has container.\n",
    "def CheckPos(Environment, Size, X, Y, Z):\n",
    "    if not ExceedsLot(Size, X, Y, Z):\n",
    "        return Environment[X, Y, Z] != '0'\n",
    "    else:\n",
    "        return False\n",
    "\n",
    "#Checks if coordinates exceed lot.\n",
    "def ExceedsLot(Size, X, Y, Z):\n",
    "    return Size[2] <= Z or Size[1] <= Y or Size[0] <= X or Z < 0 or Y < 0 or X < 0\n",
    "\n",
    "#PAS AAN, bij 4 of 5 breed kan er nog een container tussen gezet worden\n",
    "#Check whether container is in the middle of other containers.\n",
    "def CheckMiddle(Environment, Size, X, Y, Z):\n",
    "    return CheckPos(Environment, Size, X, Y - 1, 0) and CheckPos(Environment, Size, X, Y + 1, 0)\n",
    "\n",
    "#Adds container to environment using state.\n",
    "def AddContainer(Environment, Size, State, X, Y, Z):\n",
    "    tempEnvironment = np.copy(Environment)\n",
    "    if CheckBelow(Environment, X, Y, Z) and not CheckPos(Environment, Size, X, Y, Z) and not ExceedsLot(Size, X, Y, Z) and not CheckMiddle(Environment, Size, X, Y, Z):\n",
    "        tempEnvironment[X,Y,Z] = State\n",
    "    return tempEnvironment\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "77f5035e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Scores board based on priority :)\n",
    "#def scoreEnvironment(Environment):\n",
    "#    result = 0\n",
    "#    for x in range(Environment.shape[0]):\n",
    "#        for y in range(Environment.shape[1]):\n",
    "#            for z in range(Environment.shape[2]):\n",
    "#                if(Environment[x,y,z] == 'H'):\n",
    "#                    #print(\"yes\")\n",
    "#                    #result += (Environment[x,y,z+1:] == 'L').sum()\n",
    "#                    north = (Environment[x,y:,:] == 'L').sum()\n",
    "#                    south = (Environment[x,:y+1,:] == 'L').sum()\n",
    "#                    \n",
    "#                    result += -(north + south)\n",
    "#    return result\n",
    "\n",
    "\n",
    "def scoreEnvironment(Environment):\n",
    "    result = 0\n",
    "    for x in range(Environment.shape[0]):\n",
    "        for y in range(Environment.shape[1]):\n",
    "            for z in range(Environment.shape[2]):\n",
    "                result = (Environment[0,:,:] == 'L').sum()\n",
    "                result -= (Environment[0,:,:] == 'H').sum()\n",
    "                result -= (Environment[2,:,:] == 'L').sum()\n",
    "                result += (Environment[2,:,:] == 'H').sum()\n",
    "    return result\n",
    "\n",
    "\n",
    "#Returns cleared environment\n",
    "def clearEnvironment(Environment):\n",
    "    return np.array([[['0'] * Size[2]] * Size[1]] * Size[0])\n",
    "\n",
    "#Generate status of board in Boolean array\n",
    "def GenerateStatusDataset(Environment,dataType):\n",
    "    return np.concatenate([np.array([dataType == 'H'],dtype=np.float32),np.array(Environment == 'L',dtype=np.float32).flatten(), np.array(Environment == 'H', dtype=np.float32).flatten(), np.array(Environment == '0',dtype=np.float32).flatten()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "1fedc890",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generateEnvironment(size=[3,3,2]):\n",
    "    c =  [[['0'] * size[2]] * size[1]] * size[0]\n",
    "    cSize = size[0]*size[1]*size[2]*3\n",
    "    actionSize = size[0]*size[1]*size[2]\n",
    "    speelveld = np.array(c)\n",
    "    return speelveld, size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "c776193e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[['0', '0'],\n",
       "         ['0', '0'],\n",
       "         ['0', '0']],\n",
       " \n",
       "        [['0', '0'],\n",
       "         ['0', '0'],\n",
       "         ['0', '0']],\n",
       " \n",
       "        [['0', '0'],\n",
       "         ['0', '0'],\n",
       "         ['0', '0']]], dtype='<U1'),\n",
       " [3, 3, 2])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generateEnvironment()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9bc5b75e",
   "metadata": {},
   "source": [
    "# Rewards and Punishments\n",
    "rewards:\n",
    "- containers met zelfde prio dicht bij elkaar\n",
    "- stacks zo laag mogelijk houden\n",
    "\n",
    "punishments:\n",
    "- aantal moves om van de eindopstelling alle containers van één soort prio te verkrijgen\n",
    "- container met lage prio gestapeld op een container met hoge prio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a71cdacc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#environ: wat gebeurt er wanneer zet wordt gekozen waar al container staat\n",
    "#environ: game over als schip list leeg is \n",
    "#nieuwe functie: bereken score op eind door uitladingszetten te tellen van een prioriteit\n",
    "#checkmiddle functie: fiks bij 4 en 5 breed dat die niet tussen kan\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c5cc1b8e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using gpu 0\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import math\n",
    "import random\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6cfef410",
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "9ded0b4e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0, 0, 0)\n"
     ]
    }
   ],
   "source": [
    "#Zet de X,Y,Z coördinaten om naar index in array\n",
    "def coord_to_ind(X, Y, Z, lot_size):\n",
    "    grid = np.arange(np.prod(lot_size)).reshape(lot_size)\n",
    "    ind = grid[X][Y][Z]\n",
    "    return ind\n",
    "\n",
    "#Zet de index in array om naar X,Y,Z coördinaten\n",
    "def ind_to_coord(ind, lot_size):\n",
    "    grid = np.arange(np.prod(lot_size)).reshape(lot_size)\n",
    "    coord = np.where(grid[:,:,:] == ind)\n",
    "    X = coord[0][0]\n",
    "    Y = coord[1][0]\n",
    "    Z = coord[2][0]\n",
    "    return X, Y, Z\n",
    "\n",
    "c = ind_to_coord(0, [1,1,1])\n",
    "e = generateEnvironment()\n",
    "a = AddContainer(e[0],e[1],1,c[0],c[1],c[2])\n",
    "print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "54f2ab36",
   "metadata": {},
   "outputs": [],
   "source": [
    "class environ:\n",
    "    def __init__(self, lot_size=generateEnvironment()[1]):\n",
    "        if not isinstance(lot_size, (list, tuple)) or not len(lot_size) == 3:\n",
    "            raise ValueError('UNACCEPTABLE LOT_SIZE >:(')\n",
    "            \n",
    "        self.lot_size = lot_size\n",
    "        self.max_x = lot_size[0]-1\n",
    "        self.max_y = lot_size[1]-1\n",
    "        self.max_z = lot_size[2]-1\n",
    "        self.speelveld = generateEnvironment()[0]\n",
    "        self.speelveld_ind = np.prod(lot_size)\n",
    "        \n",
    "    def step(self, act):#'act' moet int zijn\n",
    "        \n",
    "        #reward = 0\n",
    "        act_loc = ind_to_coord(act, self.lot_size)\n",
    "        new_speelveld = AddContainer(self.speelveld, self.lot_size, random.randint(1,3), act_loc[0], act_loc[1], act_loc[2])\n",
    "        \n",
    "        #game_over als lot vol is, er niets meer bij kan of als schip is uitgeladen\n",
    "        #NOTE: game_over als schiplist is leeg\n",
    "        gameover = True\n",
    "        for a in range(self.max_x+1):\n",
    "            if new_speelveld[a][0][self.max_z] == '0' or new_speelveld[a][self.max_y][self.max_z] == '0':\n",
    "                gameover = False\n",
    "                break\n",
    "        \n",
    "        \n",
    "        if not gameover:\n",
    "            #container ergens zetten waar kan\n",
    "            if not (self.speelveld == new_speelveld).all():\n",
    "                reward = 1\n",
    "                self.speelveld = new_speelveld\n",
    "            #container ergens zetten waar niet kan\n",
    "            else:\n",
    "                reward = -1\n",
    "                self.speelveld = new_speelveld\n",
    "        #geen container kan meer gezet worden op lot\n",
    "        else:\n",
    "            #containerlot zit vol\n",
    "            #bij een schip met minder containers dan lotruimte: Gebruik invaarshiplist als voorwaarde\n",
    "            if not '0' in new_speelveld:\n",
    "                reward = 3\n",
    "                self.reset()\n",
    "            #containerlot had nog plek\n",
    "            else:\n",
    "                reward = -3\n",
    "                self.reset()\n",
    "            \n",
    "        return new_speelveld, reward, gameover #array, int, bool\n",
    "            \n",
    "    def reset(self):\n",
    "        \n",
    "        self.speelveld = generateEnvironment()[0]\n",
    "        return self.speelveld\n",
    "    #def render(self):\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4f017675",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReplayBuffer():\n",
    "    def __init__(self, max_size=10000):\n",
    "        self.storage = []\n",
    "        self.max_size = max_size\n",
    "        self.ptr = 0\n",
    "\n",
    "    def add(self, data):\n",
    "        if len(self.storage) == self.max_size:\n",
    "            self.storage[int(self.ptr)] = data\n",
    "            self.ptr = (self.ptr + 1) % self.max_size\n",
    "        else:\n",
    "            self.storage.append(data)\n",
    "\n",
    "    def sample(self, batch_size):\n",
    "        ind = np.random.randint(0, len(self.storage), size=batch_size)\n",
    "        x, y, u, r, d = [], [], [], [], []\n",
    "\n",
    "        for i in ind: \n",
    "            X, Y, U, R, D = self.storage[i]\n",
    "            x.append(np.array(X, copy=False))\n",
    "            y.append(np.array(Y, copy=False))\n",
    "            u.append(np.array(U, copy=False))\n",
    "            r.append(np.array(R, copy=False))\n",
    "            d.append(np.array(D, copy=False))\n",
    "\n",
    "        return np.array(x), np.array(y), np.array(u).reshape(-1,1), np.array(r).reshape(-1,1), np.array(d).reshape(-1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0420ec8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "class NeurNet(nn.Module):\n",
    "    def __init__(self, inputsize, outputsize, hiddensize):\n",
    "        super(NeurNet, self).__init__()\n",
    "        self.denselayer_1 = nn.Linear(inputsize, hiddensize)\n",
    "        self.denselayer_2 = nn.Linear(hiddensize, hiddensize)\n",
    "        self.output = nn.Linear(hiddensize, outputsize)\n",
    "        \n",
    "        #self.loss = nn.MSEloss()\n",
    "        #self.optimizer = optim.Adam(self.parameters(), lr=lrate)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.denselayer_1(x))\n",
    "        x = F.relu(self.denselayer_2(x))\n",
    "        return self.output(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "176de508",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Agent:\n",
    "    def __init__(self, inputsize, outputsize, hiddensize, lot_space, lrate):\n",
    "        self.actionspace = [i for i in range(lot_space)]\n",
    "        self.outputsize = outputsize\n",
    "        self.train_net = NeurNet(inputsize, outputsize, hiddensize).to(device)\n",
    "        self.target_net = NeurNet(inputsize, outputsize, hiddensize).to(device)\n",
    "        self.target_net.load_state_dict(self.train_net.state_dict())\n",
    "        self.optimizer = optim.Adam(self.train_net.parameters(), lr=lrate)\n",
    "        \n",
    "    def action(self, state, epsilon):\n",
    "        if np.random.rand() <= epsilon:\n",
    "            act = random.choice(self.actionspace)\n",
    "        else:\n",
    "            with torch.no_grad():\n",
    "                inputstate = torch.FloatTensor(state).to(device)\n",
    "                act = self.train_net(inputstate).max(0)[1].view(-1)\n",
    "                act = int(action)\n",
    "        return act #index van plaats in array waar container wordt gezet (int)\n",
    "    \n",
    "    def train(self, replay_buffer, batch_size, discount):\n",
    "        st_nu, st_next, act, rew, done = replay_buffer.sample(batch_size)\n",
    "        \n",
    "        state_batch = torch.FloatTensor(st_nu).to(device)\n",
    "        next_state_batch = torch.FloatTensor(st_next).to(device)\n",
    "        action_batch = torch.LongTensor(act).to(device)\n",
    "        reward_batch = torch.FloatTensor(rew).to(device)\n",
    "        done_batch = torch.FloatTensor(1 - done).to(device)\n",
    "        \n",
    "        train_q = self.train_net(state_batch).gather(1, action_batch)\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            target_net_q = reward_batch + done_batch * discount * \\\n",
    "                     torch.max( self.target_net(next_state_batch).detach(), dim=1)[0].view(batch_size, -1)\n",
    "        \n",
    "        loss_func = nn.MSELoss()\n",
    "        loss = loss_func(train_q, target_net_q)\n",
    "        self.optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        self.optimizer.step()\n",
    "        \n",
    "        return loss.detach().cpu().numpy()\n",
    "    \n",
    "    def update_netw(self, num_iter, update_every):\n",
    "        if num_iter % update_every == 0:\n",
    "            self.target_net.load_state_dict(self.train_net.state_dict())\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "5b05a050",
   "metadata": {},
   "outputs": [],
   "source": [
    "lot_size = generateEnvironment()[1]\n",
    "env = environ(lot_size)\n",
    "inputsize = lot_size[0] * lot_size[1] * lot_size[2]\n",
    "outputsize = lot_size[0] * lot_size[1] * lot_size[2]\n",
    "lotspace = lot_size[0] * lot_size[1] * lot_size[2]\n",
    "\n",
    "hidden_size = 64\n",
    "episodes = 1000\n",
    "start_train_after = 100\n",
    "learning_rate = 0.0005\n",
    "update_target_by = 1000\n",
    "batch_size = 64\n",
    "discount_gamma = 0.9\n",
    "\n",
    "epsilon_start = 1.0\n",
    "epsilon_min = 0.01\n",
    "epsilon_decay_by = episodes * 0.15\n",
    "epsilon_decrease = (epsilon_start - epsilon_min)/(epsilon_decay_by)\n",
    "\n",
    "replay_size = 15000\n",
    "replay_buffer = ReplayBuffer(max_size=replay_size)\n",
    "\n",
    "agent = Agent(inputsize,outputsize,hidden_size,lotspace,learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "49549775",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_9561/4183677267.py:26: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
      "  return np.array(x), np.array(y), np.array(u).reshape(-1,1), np.array(r).reshape(-1,1), np.array(d).reshape(-1,1)\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "can't convert np.ndarray of type numpy.object_. The only supported types are: float64, float32, float16, complex64, complex128, int64, int32, int16, int8, uint8, and bool.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn [125], line 48\u001b[0m\n\u001b[1;32m     46\u001b[0m state \u001b[38;5;241m=\u001b[39m next_state\n\u001b[1;32m     47\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m timesteps \u001b[38;5;241m>\u001b[39m start_train_after:\n\u001b[0;32m---> 48\u001b[0m     stats_loss \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[43magent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreplay_buffer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdiscount_gamma\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     49\u001b[0m     agent\u001b[38;5;241m.\u001b[39mupdate_netw(timesteps, update_target_by)\n",
      "Cell \u001b[0;32mIn [119], line 23\u001b[0m, in \u001b[0;36mAgent.train\u001b[0;34m(self, replay_buffer, batch_size, discount)\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mtrain\u001b[39m(\u001b[38;5;28mself\u001b[39m, replay_buffer, batch_size, discount):\n\u001b[1;32m     21\u001b[0m     st_nu, st_next, act, rew, done \u001b[38;5;241m=\u001b[39m replay_buffer\u001b[38;5;241m.\u001b[39msample(batch_size)\n\u001b[0;32m---> 23\u001b[0m     state_batch \u001b[38;5;241m=\u001b[39m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mFloatTensor\u001b[49m\u001b[43m(\u001b[49m\u001b[43mst_nu\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mto(device)\n\u001b[1;32m     24\u001b[0m     next_state_batch \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mFloatTensor(st_next)\u001b[38;5;241m.\u001b[39mto(device)\n\u001b[1;32m     25\u001b[0m     action_batch \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mLongTensor(act)\u001b[38;5;241m.\u001b[39mto(device)\n",
      "\u001b[0;31mTypeError\u001b[0m: can't convert np.ndarray of type numpy.object_. The only supported types are: float64, float32, float16, complex64, complex128, int64, int32, int16, int8, uint8, and bool."
     ]
    }
   ],
   "source": [
    "list_stat_rewards = []\n",
    "stats_by = 10\n",
    "total_reward = 0\n",
    "timesteps = 0\n",
    "episode_len = 0\n",
    "epsilon = epsilon_start\n",
    "\n",
    "for ep in range(episodes):\n",
    "    state = env.reset()\n",
    "    state_loss = 0\n",
    "    \n",
    "    if len(list_stat_rewards) > stats_by and np.mean(list_stat_rewards[-stats_by:],axis=0)[1] > 190:\n",
    "        print(\"episode {} with avg reward of {} in last {} episodes\").format(ep, np.mean(list_stat_rewards[-stats_by:],axis=0)[1],stats_by)\n",
    "        break\n",
    "        \n",
    "    while True:\n",
    "        timesteps += 1\n",
    "        action = agent.action(state, epsilon)\n",
    "        next_state, reward, done = env.step(action)\n",
    "        total_reward += reward\n",
    "        episode_len += 1\n",
    "        \n",
    "        if done:\n",
    "            next_state = np.zeros(state.shape)\n",
    "            reward = -1\n",
    "            replay_buffer.add((state, next_state, action, reward, done))\n",
    "            list_stat_rewards.append((ep, total_reward, episode_len))\n",
    "            total_reward = 0\n",
    "            episode_len = 0\n",
    "            \n",
    "            epsilon -= epsilon_decrease\n",
    "            if epsilon < epsilon_min:\n",
    "                epsilon = epsilon_min\n",
    "                \n",
    "            if timesteps > start_train_after and ep % stats_by == 0:\n",
    "                print('Episode: {}'.format(ep),\n",
    "                    'Timestep: {}'.format(timesteps),\n",
    "                    'Total reward: {:.1f}'.format(np.mean(list_stat_rewards[-stats_every:],axis=0)[1]),\n",
    "                    'Episode length: {:.1f}'.format(np.mean(list_stat_rewards[-stats_every:],axis=0)[2]),\n",
    "                    'Epsilon: {:.2f}'.format(epsilon),\n",
    "                    'Loss: {:.3f}'.format(stats_loss))\n",
    "            break\n",
    "        else:\n",
    "            replay_buffer.add((state, next_state, action, reward, done))\n",
    "            \n",
    "        state = next_state\n",
    "        if timesteps > start_train_after:\n",
    "            \n",
    "            stats_loss += agent.train(replay_buffer, batch_size, discount_gamma)\n",
    "            agent.update_netw(timesteps, update_target_by)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "047f7a5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "list_stat_rewards = []\n",
    "stats_by = 10\n",
    "total_reward = 0\n",
    "timesteps = 0\n",
    "episode_len = 0\n",
    "epsilon = epsilon_start\n",
    "\n",
    "for ep in range(episodes):\n",
    "    state = env.reset()\n",
    "    state_loss = 0\n",
    "    \n",
    "    if len(list_stat_rewards) > stats_by and np.mean(list_stat_rewards[-stats_by:],axis=0)[1] > 190:\n",
    "        print(\"episode {} with avg reward of {} in last {} episodes\").format(ep, np.mean(list_stat_rewards[-stats_by:],axis=0)[1],stats_by)\n",
    "        break\n",
    "        \n",
    "    while True:\n",
    "        timesteps += 1\n",
    "        action = agent.action(state, epsilon)\n",
    "        next_state, reward, done = env.step(action)\n",
    "        total_reward += reward\n",
    "        episode_len += 1\n",
    "        \n",
    "        if done:\n",
    "            next_state = np.zeros(state.shape)\n",
    "            reward = -1\n",
    "            replay_buffer.add((state, next_state, action, reward, done))\n",
    "            list_stat_rewards.append((ep, total_reward, episode_len))\n",
    "            total_reward = 0\n",
    "            episode_len = 0\n",
    "            \n",
    "            epsilon -= epsilon_decrease\n",
    "            if epsilon < epsilon_min:\n",
    "                epsilon = epsilon_min\n",
    "                \n",
    "            if timesteps > start_train_after and ep % stats_by == 0:\n",
    "                print('Episode: {}'.format(ep),\n",
    "                    'Timestep: {}'.format(timesteps),\n",
    "                    'Total reward: {:.1f}'.format(np.mean(list_stat_rewards[-stats_every:],axis=0)[1]),\n",
    "                    'Episode length: {:.1f}'.format(np.mean(list_stat_rewards[-stats_every:],axis=0)[2]),\n",
    "                    'Epsilon: {:.2f}'.format(epsilon),\n",
    "                    'Loss: {:.3f}'.format(stats_loss))\n",
    "            break\n",
    "        else:\n",
    "            replay_buffer.add((state, next_state, action, reward, done))\n",
    "            \n",
    "        state = next_state\n",
    "        if timesteps > start_train_after:\n",
    "            stats_loss += agent.train(replay_buffer, batch_size, discount_gamma)\n",
    "            agent.update_netw(timesteps, update_target_by)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2064e0df",
   "metadata": {},
   "source": [
    "# Testgebied"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "4621f6ef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.01551167, -0.01107467,  0.00019003, -0.01944593])"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.uniform(low=-0.02, high=0.02, size=(4,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "aeeb044a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1, 2, 3}"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dicts = {'a':3,'b':2,'c':1,'d':3,'e':2,'f':1}\n",
    "set(dicts.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "68ac4868",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[['0' '0' '0']\n",
      "  ['2' '2' '2']]]\n",
      "({0: '0', 1: '0', 2: '0'}, {0: '0', 1: '0', 2: '0', 3: '2', 4: '2', 5: '2'})\n",
      "{3: '2', 4: '2', 5: '2', 0: '0', 1: '0', 2: '0'}\n",
      "{}\n",
      "{4: '2', 5: '2', 0: '0', 1: '0', 2: '0'}\n",
      "{}\n",
      "{5: '2', 0: '0', 1: '0', 2: '0'}\n",
      "{}\n",
      "9.0\n"
     ]
    }
   ],
   "source": [
    "lotsize = [1,2,3]\n",
    "def vulenv(lot, index, prio):\n",
    "    act_loc = ind_to_coord(index, lotsize)\n",
    "    new_lot = AddContainer(lot, lotsize, prio, act_loc[0], act_loc[1], act_loc[2])\n",
    "    return new_lot\n",
    "\n",
    "lot = generateEnvironment(lotsize)[0]\n",
    "lot = vulenv(lot, 0, 0)\n",
    "lot = vulenv(lot, 1, 0)\n",
    "lot = vulenv(lot, 2, 0)\n",
    "lot = vulenv(lot, 3, 2)\n",
    "lot = vulenv(lot, 4, 2)\n",
    "lot = vulenv(lot, 5, 2)\n",
    "# lot = vulenv(lot, 6, 0)\n",
    "# lot = vulenv(lot, 7, 0)\n",
    "# lot = vulenv(lot, 8, 0)\n",
    "# lot = vulenv(lot, 9, 3)\n",
    "# lot = vulenv(lot, 10, 2)\n",
    "# lot = vulenv(lot, 11, 1)\n",
    "# lot = vulenv(lot, 12, 1)\n",
    "# lot = vulenv(lot, 13, 1)\n",
    "# lot = vulenv(lot, 14, 2)\n",
    "# lot = vulenv(lot, 15, 2)\n",
    "# lot = vulenv(lot, 16, 3)\n",
    "# lot = vulenv(lot, 17, 3)\n",
    "print(lot)\n",
    "\n",
    "rs = cont_routes(lot, lotsize,0)\n",
    "print(rs)\n",
    "r1 = rs[0]\n",
    "r2 = rs[1]\n",
    "#print(route_info(r1))\n",
    "#print(route_info(r2))\n",
    "print(score_eindveld(lot,lotsize))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "bd3a219e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Geeft van een gegeven containerindex in de lotindeling de twee wegen om deze container te bereiken\n",
    "def cont_routes(lot_indeling, lot_size, ind): #lot_indeling: array, lot_size: [x,y,z], ind: int\n",
    "    X = lot_size[0]\n",
    "    Y = lot_size[1]\n",
    "    Z = lot_size[2]\n",
    "    lot_index = np.arange(np.prod(lot_size)).reshape(lot_size) #lotindeling maar dan de indexes\n",
    "    indc = ind_to_coord(ind, lot_size) #vind coördinaten van ingevoerde index\n",
    "    \n",
    "    #voeg de containerstack waar de betreffende container in zit toe aan de dictionary als start van de route\n",
    "    stackvalue = lot_indeling[indc[0]][indc[1]]\n",
    "    stackkey = lot_index[indc[0]][indc[1]]\n",
    "    stackdict = dict(zip(stackkey,stackvalue))\n",
    "    \n",
    "    #route 1 berekenen\n",
    "    routestart = {k: v for k, v in stackdict.items() if k >= ind}\n",
    "    route1 = routestart\n",
    "    i = 0\n",
    "    while (lot_index[indc[0]][indc[1]-i][0] % (Y*Z)) > 0:\n",
    "        for z in range(Z):\n",
    "            route1[lot_index[indc[0]][indc[1]-(i+1)][z]] = lot_indeling[indc[0]][indc[1]-(i+1)][z]\n",
    "        i += 1\n",
    "    \n",
    "    #route 2 berekenen\n",
    "    routestart = {k: v for k, v in stackdict.items() if k >= ind}\n",
    "    route2 = routestart\n",
    "    i = 0\n",
    "    while (lot_index[indc[0]][indc[1]+i][Z-1] % (Y*Z)) < (Y*Z-1):\n",
    "        for z in range(Z):\n",
    "            route2[lot_index[indc[0]][indc[1]+(i+1)][z]] = lot_indeling[indc[0]][indc[1]+(i+1)][z]\n",
    "        i += 1\n",
    "    \n",
    "    return route1, route2 #route1: dictionary, route2: dictionary\n",
    "\n",
    "\n",
    "#Geeft de score van een indeling terug\n",
    "def score_eindveld(lot_indeling,lot_size):\n",
    "    X = lot_size[0]\n",
    "    Y = lot_size[1]\n",
    "    Z = lot_size[2]\n",
    "    lot_index = np.arange(np.prod(lot_size)).reshape(lot_size)\n",
    "    \n",
    "    #maak een lijst van alle verschillende prio indexen in de indeling\n",
    "    #'0' geeft een lege plek aan, dus die telt niet mee\n",
    "    if np.unique(lot_indeling)[0] == '0':\n",
    "        lot_unique = np.unique(lot_indeling)[1:]\n",
    "    else:\n",
    "        lot_unique = np.unique(lot_indeling)\n",
    "\n",
    "    score = 0\n",
    "    \n",
    "    #beloon 0 rijen alleen bij een X >= 1, Y >= 3, Z >= 1\n",
    "    \n",
    "    \n",
    "    for p in lot_unique:\n",
    "    \n",
    "        obstaclesdict = {}\n",
    "        for i in range(len(np.where(lot_indeling == p)[0])):\n",
    "            x = np.where(lot_indeling == p)[0][i]\n",
    "            y = np.where(lot_indeling == p)[1][i]\n",
    "            z = np.where(lot_indeling == p)[2][i]\n",
    "            routes = cont_routes(lot_indeling, lot_size, lot_index[x][y][z])\n",
    "            routelen1 = len({k: v for k,v in routes[0].items() if int(v) > int(p)})\n",
    "            routelen2 = len({k: v for k,v in routes[1].items() if int(v) > int(p)})\n",
    "\n",
    "            #kies eerste route als die korter is dan tweede\n",
    "            if routelen1 < routelen2:\n",
    "                shortroute = routes[0]\n",
    "            #kies tweede route als die korter is dan eerste\n",
    "            elif routelen1 > routelen2:\n",
    "                shortroute = routes[1]\n",
    "            #als beide routes even lang zijn, kijk dan hoeveel containers op de route dezelfde prio klasse hebben\n",
    "            else:\n",
    "                sameclassroute1 = sum(1 for v in routes[0].values() if v <= list(routes[0].values())[0])\n",
    "                sameclassroute2 = sum(1 for v in routes[1].values() if v <= list(routes[1].values())[0])\n",
    "                #kies eerste route als aantal dezelfe prio klasse containers gelijk aan of meer is dan tweede\n",
    "                if sameclassroute1 >= sameclassroute2:\n",
    "                    shortroute = routes[0]\n",
    "                #kies tweede route als aantal dezelfe prio klasse containers meer is dan eerste\n",
    "                else:\n",
    "                    shortroute = routes[1]\n",
    "            print(shortroute)\n",
    "\n",
    "            #sla alle containers op die op de route in de weg zitten (de containers met een hogere prioklasse)\n",
    "            for c in shortroute:\n",
    "                if int(shortroute[c]) > int(p):\n",
    "                    obstaclesdict[c] = shortroute[c]\n",
    "            print(obstaclesdict)\n",
    "            \n",
    "            #score omlaag als containers in de weg zitten\n",
    "            if len(obstaclesdict) > 0:\n",
    "                for o in obstaclesdict:\n",
    "                    score = score - (int(obstaclesdict[o]) - int(p)) \n",
    "                    #score gebaseerd op prio index van de blokkerende container, AANPASBAAR\n",
    "                    \n",
    "            #score omhoog als geen containers in de weg zitten\n",
    "            else:\n",
    "                score += ((Y*Z)/2) \n",
    "                #score gebaseerd op lotsize, AANPASBAAR\n",
    "    \n",
    "    return score #score: integer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "78b98527",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']\n",
      "  ['1' '0']]]\n",
      "---------------------------------\n",
      "[[['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['0' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '0']]]\n",
      "---------------------------------\n",
      "[[['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['0' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '0']]]\n",
      "---------------------------------\n",
      "[[['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '1']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['0' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '0']]]\n",
      "---------------------------------\n",
      "[[['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '1']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['0' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '0']]]\n",
      "---------------------------------\n",
      "[[['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '1']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['1' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['0' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '0']]]\n",
      "---------------------------------\n",
      "[[['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '1']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['1' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]]\n",
      "---------------------------------\n",
      "[[['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '1']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['1' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]]\n",
      "---------------------------------\n",
      "[[['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '1']\n",
      "  ['0' '1']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['1' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]]\n",
      "---------------------------------\n",
      "[[['0' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '1']\n",
      "  ['0' '1']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['1' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]]\n",
      "---------------------------------\n",
      "[[['1' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '1']\n",
      "  ['0' '1']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['1' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]]\n",
      "---------------------------------\n",
      "[[['1' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '1']\n",
      "  ['0' '1']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['1' '0']\n",
      "  ['0' '0']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]]\n",
      "---------------------------------\n",
      "[[['1' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['1' '0']\n",
      "  ['0' '0']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]]\n",
      "---------------------------------\n",
      "[[['1' '0']\n",
      "  ['0' '0']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['1' '0']\n",
      "  ['0' '0']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]]\n",
      "---------------------------------\n",
      "[[['1' '0']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['1' '0']\n",
      "  ['0' '0']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]]\n",
      "---------------------------------\n",
      "[[['1' '0']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['1' '0']\n",
      "  ['0' '0']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]]\n",
      "---------------------------------\n",
      "[[['1' '0']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['1' '0']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '0']\n",
      "  ['1' '0']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['0' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '0']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['0' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['1' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['1' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['1' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['1' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '0']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n",
      "[[['1' '1']\n",
      "  ['1' '0']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['0' '1']\n",
      "  ['1' '1']\n",
      "  ['0' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]\n",
      "\n",
      " [['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']\n",
      "  ['1' '1']]]\n",
      "---------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_x = generateEnvironment()[1][0]\n",
    "max_y = generateEnvironment()[1][1]\n",
    "max_z = generateEnvironment()[1][2]\n",
    "        \n",
    "speelveld = generateEnvironment()[0]\n",
    "done = False\n",
    "        \n",
    "while not done:\n",
    "    rnx = random.randrange(max_x)\n",
    "    rny = random.randrange(max_y)\n",
    "    rnz = random.randrange(max_z)\n",
    "    speelveld[rnx][rny][rnz] = '1'\n",
    "    print(speelveld)\n",
    "    print('---------------------------------')\n",
    "    done = True\n",
    "    for a in range(max_x):\n",
    "        if speelveld[a][0][max_z-1] == '0' or speelveld[a][max_y-1][max_z-1] == '0':\n",
    "            done = False\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "a5ea58a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<TimeLimit<CartPoleEnv<CartPole-v0>>>"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "f052caf4",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'high' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn [30], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m gym\u001b[38;5;241m.\u001b[39mspaces\u001b[38;5;241m.\u001b[39mBox(\u001b[38;5;241m-\u001b[39m\u001b[43mhigh\u001b[49m, high, dtype\u001b[38;5;241m=\u001b[39mnp\u001b[38;5;241m.\u001b[39mfloat32)\n",
      "\u001b[0;31mNameError\u001b[0m: name 'high' is not defined"
     ]
    }
   ],
   "source": [
    "gym.spaces.Box(-high, high, dtype=np.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f2d0a62",
   "metadata": {},
   "source": [
    "# Werkend model op cartpole"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "db60c174",
   "metadata": {},
   "outputs": [],
   "source": [
    "class FeedForwardNeuralNet(nn.Module):\n",
    "    def __init__(self, state_size, action_size, hidden_size):\n",
    "        super(FeedForwardNeuralNet, self).__init__()\n",
    "        self.dense_layer_1 = nn.Linear(state_size, hidden_size)\n",
    "        self.dense_layer_2 = nn.Linear(hidden_size, hidden_size)\n",
    "        self.output = nn.Linear(hidden_size, action_size)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.dense_layer_1(x))\n",
    "        x = F.relu(self.dense_layer_2(x))\n",
    "        return self.output(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "013f0826",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CartPoleAgent():\n",
    "    def __init__(self, state_size, action_size, hidden_size, learning_rate ):\n",
    "        \n",
    "        self.action_size = action_size\n",
    "        self.train_net = FeedForwardNeuralNet(state_size, action_size, hidden_size).to(device)\n",
    "        self.target_net = FeedForwardNeuralNet(state_size, action_size, hidden_size).to(device)\n",
    "        self.target_net.load_state_dict(self.train_net.state_dict())\n",
    "        self.optimizer = optim.Adam(self.train_net.parameters(), lr=learning_rate)\n",
    "\n",
    "        \n",
    "    def select_action(self, state, epsilon):\n",
    "        # select action according to epsilon-greedy method\n",
    "        if np.random.rand() <= epsilon:\n",
    "            action = env.action_space.sample()\n",
    "        else:\n",
    "            # greedy action is the largest Q value from the train network based on the input\n",
    "            with torch.no_grad():\n",
    "                input_state = torch.FloatTensor(state).to(device)\n",
    "                # feed input into the network and get the max action value\n",
    "                action = self.train_net(input_state).max(0)[1].view(-1)\n",
    "                action = int(action)\n",
    "        return action\n",
    "\n",
    "    def train(self, replay_buffer, batch_size, discount):\n",
    "        # train the training network\n",
    "        # sample a batch from the replay buffer\n",
    "        x0, x1, a, r, d = replay_buffer.sample(batch_size)\n",
    "\n",
    "        # turn batches into tensors and attack to GPU if available\n",
    "        state_batch = torch.FloatTensor(x0).to(device)\n",
    "        next_state_batch = torch.FloatTensor(x1).to(device)\n",
    "        action_batch = torch.LongTensor(a).to(device)\n",
    "        reward_batch = torch.FloatTensor(r).to(device)\n",
    "        done_batch = torch.FloatTensor(1 - d).to(device)\n",
    "\n",
    "        # get train net Q values\n",
    "        train_q = self.train_net(state_batch).gather(1, action_batch)\n",
    "        \n",
    "        # get target net Qvalues\n",
    "        with torch.no_grad():\n",
    "            target_net_q = reward_batch + done_batch * discount * \\\n",
    "                     torch.max( self.target_net(next_state_batch).detach(), dim=1)[0].view(batch_size, -1)\n",
    "\n",
    "        # create loss function\n",
    "        loss_fn = nn.MSELoss()\n",
    "        # get loss between train q values and target q values\n",
    "        loss = loss_fn(train_q, target_net_q)\n",
    "        # optimize the parameters with the loss\n",
    "        self.optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        self.optimizer.step()\n",
    "        # we return the loss so we can monitor loss and debug the network if necessary\n",
    "        return loss.detach().cpu().numpy()\n",
    "    \n",
    "    \n",
    "    def update_target_network(self, num_iter, update_every):\n",
    "        # update target network every so often\n",
    "        # hard target network update: updates target network fully with train network params\n",
    "        if num_iter % update_every == 0:\n",
    "            self.target_net.load_state_dict(self.train_net.state_dict())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "54741b92",
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize environment\n",
    "env = gym.make('CartPole-v0')\n",
    "action_size = env.action_space.n\n",
    "state_size = env.observation_space.shape[0]\n",
    "\n",
    "# set seed\n",
    "seed = 31\n",
    "env.seed(seed)\n",
    "np.random.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "if torch.cuda.is_available():\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.cuda.manual_seed_all(seed)\n",
    "\n",
    "# hyperparameters\n",
    "episodes = 1000 # run agent for this many episodes\n",
    "start_training_after = 100 # collect data for this many timesteps before training\n",
    "hidden_size = 64 # number of units in NN hidden layers\n",
    "learning_rate = 0.0005 # learning rate for optimizer\n",
    "update_target_every_ts = 1000 # update target network after this many steps\n",
    "batch_size = 64 # mini batch size we train network on\n",
    "discount = 0.9 # gamma value\n",
    "\n",
    "epsilon_start = 1.0 # epsilon start value\n",
    "epsilon_min = 0.01  # epsilon end value\n",
    "epsilon_decay_steps = episodes * .15 # decay epsilon over this many episodes\n",
    "epsilon_step = (epsilon_start - epsilon_min)/(epsilon_decay_steps) # decrement epsilon by this amount every timestep\n",
    "\n",
    "# create replay buffer\n",
    "replay_size = 15000\n",
    "replay_buffer = ReplayBuffer(max_size=replay_size)\n",
    "\n",
    "# create cartpole agent\n",
    "cartpole_agent = CartPoleAgent(state_size, action_size, hidden_size, learning_rate)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "088f4ee3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 10 Timestep: 222 Total reward: 20.3 Episode length: 20.3 Epsilon: 0.93 Loss: 3.400\n",
      "Episode: 20 Timestep: 429 Total reward: 20.7 Episode length: 20.7 Epsilon: 0.86 Loss: 7.741\n",
      "Episode: 30 Timestep: 787 Total reward: 35.8 Episode length: 35.8 Epsilon: 0.80 Loss: 7.908\n",
      "Episode: 40 Timestep: 1056 Total reward: 26.9 Episode length: 26.9 Epsilon: 0.73 Loss: 7.011\n",
      "Episode: 50 Timestep: 1535 Total reward: 47.9 Episode length: 47.9 Epsilon: 0.66 Loss: 32.028\n",
      "Episode: 60 Timestep: 2140 Total reward: 60.5 Episode length: 60.5 Epsilon: 0.60 Loss: 4.431\n",
      "Episode: 70 Timestep: 2736 Total reward: 59.6 Episode length: 59.6 Epsilon: 0.53 Loss: 5.248\n",
      "Episode: 80 Timestep: 3501 Total reward: 76.5 Episode length: 76.5 Epsilon: 0.47 Loss: 4.845\n",
      "Episode: 90 Timestep: 4278 Total reward: 77.7 Episode length: 77.7 Epsilon: 0.40 Loss: 9.164\n",
      "Episode: 100 Timestep: 5406 Total reward: 112.8 Episode length: 112.8 Epsilon: 0.33 Loss: 10.224\n",
      "Episode: 110 Timestep: 7167 Total reward: 176.1 Episode length: 176.1 Epsilon: 0.27 Loss: 69.404\n",
      "Episode: 120 Timestep: 8535 Total reward: 136.8 Episode length: 136.8 Epsilon: 0.20 Loss: 64.318\n",
      "Episode: 130 Timestep: 10279 Total reward: 174.4 Episode length: 174.4 Epsilon: 0.14 Loss: 62.940\n",
      "Episode: 140 Timestep: 12128 Total reward: 184.9 Episode length: 184.9 Epsilon: 0.07 Loss: 57.989\n",
      "Episode: 150 Timestep: 13783 Total reward: 165.5 Episode length: 165.5 Epsilon: 0.01 Loss: 45.809\n",
      "Episode: 160 Timestep: 15386 Total reward: 160.3 Episode length: 160.3 Epsilon: 0.01 Loss: 35.216\n",
      "Episode: 170 Timestep: 16872 Total reward: 148.6 Episode length: 148.6 Epsilon: 0.01 Loss: 44.768\n",
      "Episode: 180 Timestep: 18286 Total reward: 141.4 Episode length: 141.4 Epsilon: 0.01 Loss: 41.116\n",
      "Episode: 190 Timestep: 19725 Total reward: 143.9 Episode length: 143.9 Epsilon: 0.01 Loss: 38.935\n",
      "Episode: 200 Timestep: 21475 Total reward: 175.0 Episode length: 175.0 Epsilon: 0.01 Loss: 37.525\n",
      "Episode: 210 Timestep: 23244 Total reward: 176.9 Episode length: 176.9 Epsilon: 0.01 Loss: 46.643\n",
      "Episode: 220 Timestep: 25008 Total reward: 176.4 Episode length: 176.4 Epsilon: 0.01 Loss: 40.398\n",
      "Stopping at episode 230 with average rewards of 194.0 in last 10 episodes\n"
     ]
    }
   ],
   "source": [
    "stats_rewards_list = [] # store stats for plotting in this\n",
    "stats_every = 10 # print stats every this many episodes\n",
    "total_reward = 0\n",
    "timesteps = 0\n",
    "episode_length = 0\n",
    "epsilon = epsilon_start\n",
    "\n",
    "for ep in range(episodes):\n",
    "    state = env.reset()\n",
    "    stats_loss = 0\n",
    "    \n",
    "    # stopping condition for training if agent reaches the amount of reward\n",
    "    if len(stats_rewards_list) > stats_every and np.mean(stats_rewards_list[-stats_every:],axis=0)[1] > 190:\n",
    "        print(\"Stopping at episode {} with average rewards of {} in last {} episodes\".\n",
    "            format(ep, np.mean(stats_rewards_list[-stats_every:],axis=0)[1], stats_every))\n",
    "        break  \n",
    "    \n",
    "    # train in each episode until episode is done\n",
    "    while True:\n",
    "        timesteps += 1\n",
    "        #env.render()\n",
    "        # select an action from the agent's policy\n",
    "        action = cartpole_agent.select_action(state, epsilon)\n",
    "        # enter action into the env\n",
    "        next_state, reward, done, _ = env.step(action)\n",
    "        total_reward += reward\n",
    "        episode_length += 1\n",
    "        \n",
    "        if done:\n",
    "            next_state = np.zeros(state.shape)\n",
    "            reward = -1\n",
    "            # add experience to replay buffer\n",
    "            replay_buffer.add((state, next_state, action, reward, done))\n",
    "            \n",
    "            stats_rewards_list.append((ep, total_reward, episode_length))\n",
    "            total_reward = 0\n",
    "            episode_length = 0\n",
    "            \n",
    "            epsilon -= epsilon_step\n",
    "            if epsilon < epsilon_min:\n",
    "                epsilon = epsilon_min\n",
    "                \n",
    "            if timesteps > start_training_after and ep % stats_every == 0:\n",
    "                print('Episode: {}'.format(ep),\n",
    "                    'Timestep: {}'.format(timesteps),\n",
    "                    'Total reward: {:.1f}'.format(np.mean(stats_rewards_list[-stats_every:],axis=0)[1]),\n",
    "                    'Episode length: {:.1f}'.format(np.mean(stats_rewards_list[-stats_every:],axis=0)[2]),\n",
    "                    'Epsilon: {:.2f}'.format(epsilon),\n",
    "                    'Loss: {:.3f}'.format(stats_loss))\n",
    "                \n",
    "            break\n",
    "        else:\n",
    "            # add experience to replay buffer\n",
    "            replay_buffer.add((state, next_state, action, reward, done))\n",
    "        \n",
    "        state = next_state\n",
    "        if timesteps > start_training_after:\n",
    "            # train the agent\n",
    "            stats_loss += cartpole_agent.train(replay_buffer, batch_size, discount)\n",
    "            # update the target network every (if conditions are met in update_target_network)\n",
    "            cartpole_agent.update_target_network(timesteps, update_target_every_ts)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b92d0424",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}