learner.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": 3
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python_defaultSpec_1596330170308",
   "display_name": "Python 3.7.6 64-bit"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from wumpusworld.simplified_wumpus_world import SimplifiedWumpusWorld"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = SimplifiedWumpusWorld()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "action_space_size = env.action_space.n\n",
    "state_space_size = env.observation_space.n\n",
    "\n",
    "q_table = np.zeros((state_space_size, action_space_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_episodes = 10000\n",
    "max_steps_per_episode = 100\n",
    "\n",
    "learning_rate = 0.1\n",
    "discount_rate = 0.99\n",
    "\n",
    "exploration_rate = 1.0\n",
    "max_exploration_rate = 1.0\n",
    "min_exploration_rate = 0.1\n",
    "exploration_decay_rate = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "rewards_all_episodes = []\n",
    "\n",
    "for episode in range(num_episodes):\n",
    "    state = env.reset()\n",
    "\n",
    "    rewards_current_episode = 0\n",
    "\n",
    "    for step in range(max_steps_per_episode):\n",
    "        exploration_rate_threshold = np.random.uniform(0, 1)\n",
    "        if exploration_rate_threshold > exploration_rate:\n",
    "            action = np.argmax(q_table[state, :])\n",
    "        else:\n",
    "            action = env.action_space.sample()\n",
    "\n",
    "        new_state, reward, done = env.step(action)\n",
    "\n",
    "        q_table[state, action] = q_table[state, action] * (1 - learning_rate) + learning_rate * (reward + discount_rate * np.max(q_table[new_state, :]))\n",
    "\n",
    "        state = new_state\n",
    "        rewards_current_episode += reward\n",
    "\n",
    "        if done:\n",
    "            break\n",
    "\n",
    "    exploration_rate = min_exploration_rate + (max_exploration_rate - min_exploration_rate) * np.exp(-exploration_decay_rate * episode)\n",
    "\n",
    "    rewards_all_episodes.append(rewards_current_episode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "~~~~~~~Average Rewards Per Thousand Episodes~~~~~~\n1000 : -708.4459999999999\n2000 : 136.65800000000004\n3000 : 530.712\n4000 : 640.8359999999999\n5000 : 694.7529999999999\n6000 : 730.8069999999999\n7000 : 714.8999999999999\n8000 : 718.8799999999999\n9000 : 720.8799999999999\n10000: 724.866\n\n~~~~~~~~~~~~~~~~~~~~~~Q-Table~~~~~~~~~~~~~~~~~~~~~~\n[[-1000.           935.62816434   935.62816434   946.08905489]\n [  956.655611     946.08905489   935.62816434 -1000.        ]\n [    0.             0.             0.             0.        ]\n [    0.             0.             0.             0.        ]\n [    0.             0.             0.             0.        ]\n [  967.3289       946.08905489 -1000.           967.3289    ]\n [  978.11        -999.9999652    956.6554991   -999.9996074 ]\n [    0.             0.             0.             0.        ]\n [ -999.7815255   -999.66701036   952.22823601   967.3289    ]\n [  956.655611     956.655611     956.655611     978.11      ]\n [-1000.           967.3289       967.3289       989.        ]\n [ 1000.         -1000.           978.11         989.        ]\n [    0.             0.             0.             0.        ]\n [  956.39898601   967.3289      -999.66701036  -999.82303565]\n [    0.             0.             0.             0.        ]\n [    0.             0.             0.             0.        ]]\n"
    }
   ],
   "source": [
    "rewards_per_thousand_episodes = np.split(np.array(rewards_all_episodes), num_episodes/1000)\n",
    "count = 1000\n",
    "print(\"~~~~~~~Average Rewards Per Thousand Episodes~~~~~~\")\n",
    "for r in rewards_per_thousand_episodes:\n",
    "    print(f\"{count: <5}: {np.sum(r/1000)}\")\n",
    "    count += 1000\n",
    "\n",
    "print()\n",
    "print(\"~~~~~~~~~~~~~~~~~~~~~~Q-Table~~~~~~~~~~~~~~~~~~~~~~\")\n",
    "print(q_table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "canvas = Image.open(\"images/world.png\")\n",
    "agent = Image.open(\"images/rebort-scaled.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "locations = {(x, y): (20 * (x + 1) + 500 * x, 1600 - (20 * (y + 1) + 500 * y)) for y in range(4) for x in range(4)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "Reward: 995\n"
    }
   ],
   "source": [
    "images = []\n",
    "\n",
    "agentX = 0\n",
    "agentY = 0\n",
    "\n",
    "b = canvas.copy()\n",
    "b.paste(agent, locations[(agentX, agentY)], agent)\n",
    "images.append(b)\n",
    "\n",
    "state = env.reset()\n",
    "\n",
    "rewards_current_episode = 0\n",
    "\n",
    "for step in range(max_steps_per_episode):\n",
    "    action = np.argmax(q_table[state, :])\n",
    "\n",
    "    if action == 0:\n",
    "        agentY = min(3, agentY + 1)\n",
    "    elif action == 1:\n",
    "        agentY = max(0, agentY - 1)\n",
    "    elif action == 2:\n",
    "        agentX = max(0, agentX - 1)\n",
    "    else:\n",
    "        agentX = min(3, agentX + 1)\n",
    "\n",
    "    b = canvas.copy()\n",
    "    b.paste(agent, locations[(agentX, agentY)], agent)\n",
    "    images.append(b)\n",
    "\n",
    "    new_state, reward, done = env.step(action)\n",
    "\n",
    "    state = new_state\n",
    "    rewards_current_episode += reward\n",
    "\n",
    "    if done:\n",
    "        break\n",
    "\n",
    "print(f\"Reward: {rewards_current_episode}\")\n",
    "\n",
    "images[0].save('images/anitest.gif',\n",
    "               save_all=True,\n",
    "               append_images=images[1:],\n",
    "               duration=500,\n",
    "               loop=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"images/anitest.gif\" width=\"50%\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}