sokoban_environment.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d1fe5a1f",
   "metadata": {},
   "source": [
    "# Sumit \n",
    "# 19312\n",
    "# Reinforcement Learning \n",
    "# Assignment 2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ef15f20",
   "metadata": {},
   "source": [
    "## Creating Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "87aee4f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Definition of the Warehouse Agent environment\n",
    "\"\"\"\n",
    "import numpy as np\n",
    "\n",
    "class WarehouseAgent():\n",
    "    def __init__(self):\n",
    "        \"\"\"\n",
    "        Initializing the environment\n",
    "        \"\"\"\n",
    "        self.GRID_DIM = [7,6]\n",
    "        self.agent_position = [1,2]\n",
    "        self.box_location = [4,3]\n",
    "        self.goal_location = [3,1]\n",
    "        \n",
    "        state = np.zeros((7,6,4))\n",
    "        \n",
    "        #place agent\n",
    "        state[1,2] = np.array([0,0,0,1])\n",
    "        #place walls\n",
    "        state[0,0]= np.array([0,0,1,0])\n",
    "        state[0,1]= np.array([0,0,1,0])\n",
    "        state[0,2]= np.array([0,0,1,0])\n",
    "        state[0,3]= np.array([0,0,1,0])\n",
    "        state[1,0]= np.array([0,0,1,0])\n",
    "        state[1,3]= np.array([0,0,1,0])\n",
    "        state[2,0]= np.array([0,0,1,0])\n",
    "        state[2,3]= np.array([0,0,1,0])\n",
    "        state[2,4]= np.array([0,0,1,0])\n",
    "        state[2,5]= np.array([0,0,1,0])\n",
    "        state[3,0]= np.array([0,0,1,0])\n",
    "        state[3,5]= np.array([0,0,1,0])\n",
    "        state[4,0]= np.array([0,0,1,0])\n",
    "        state[4,5]= np.array([0,0,1,0])\n",
    "        state[5,0]= np.array([0,0,1,0])\n",
    "        state[5,3]= np.array([0,0,1,0])\n",
    "        state[5,4]= np.array([0,0,1,0])\n",
    "        state[5,5]= np.array([0,0,1,0])\n",
    "        state[6,0]= np.array([0,0,1,0])\n",
    "        state[6,1]= np.array([0,0,1,0])\n",
    "        state[6,2]= np.array([0,0,1,0])\n",
    "        state[6,3]= np.array([0,0,1,0])\n",
    "\n",
    "        #place box\n",
    "        state[4,3] = np.array([0,1,0,0])\n",
    "        #place goal\n",
    "        state[3,1] = np.array([1,0,0,0])\n",
    "        self.state = state\n",
    "     \n",
    "    def reset(self):\n",
    "        \"\"\"Function to reset the environment at the end of each episode to its initial state configuration\n",
    "\n",
    "        Returns:\n",
    "            state: the state of the environment reset to its initial conditions\n",
    "        \"\"\"\n",
    "        #replace agent\n",
    "        state[1,2] = np.array([0,0,0,1])\n",
    "        #replace box\n",
    "        state[4,3] = np.array([0,1,0,0])\n",
    "\n",
    "        return state\n",
    "    \n",
    "    def getLoc(self, state, level):\n",
    "        for i in range(0,7):\n",
    "            for j in range(0,6):\n",
    "                if (state[i,j][level] == 1):\n",
    "                    return i,j\n",
    "\n",
    "    def step(self, state, action):\n",
    "        \"\"\"Function to control and evaluate the agents' action\n",
    "\n",
    "        Args:\n",
    "            action: pass on the action which the agent needs to take at that time step\n",
    "\n",
    "        Returns:\n",
    "            new_state: the new state agent reaches after taking the action\n",
    "            reward: the reward obtained on taking the action\n",
    "            done: boolean value to determine if episode terminating condition is reached\n",
    "        \"\"\"         \n",
    "        player_loc = self.getLoc(state, 3)\n",
    "        goal = self.getLoc(state, 0)\n",
    "        box = self.getLoc(state, 1)\n",
    "        \n",
    "        actions = [[-1,0],[1,0],[0,-1],[0,1]]\n",
    "        new_loc = (player_loc[0] + actions[action][0], player_loc[1] + actions[action][1])\n",
    "        new_box = (box[0] + actions[action][0], box[1] + actions[action][1])\n",
    "        if (state[new_loc][2] != 1):\n",
    "            if (state[new_loc][1] == 1 and state[new_box][2] != 1):\n",
    "                state[new_loc][3] = 1\n",
    "                state[new_box][1] = 1\n",
    "                state[player_loc][3] = 0\n",
    "                state[box][1] = 0\n",
    "            elif (state[new_loc][1] != 1):\n",
    "                state[new_loc][3] = 1\n",
    "                state[player_loc][3] = 0\n",
    "        \n",
    "        new_player_loc = self.getLoc(state, 3)\n",
    "        if (not new_player_loc):\n",
    "            state[player_loc] = np.array([0,0,0,1])\n",
    "        new_box_loc = self.getLoc(state, 1)\n",
    "        if (not new_box_loc):\n",
    "            state[box] = np.array([0,0,0,1])\n",
    "            \n",
    "        box = self.getLoc(state, 1)\n",
    "        goal = self.getLoc(state, 0)\n",
    "\n",
    "        new_state = state\n",
    "        reward = -1\n",
    "        done = False\n",
    "        if (box == goal):\n",
    "            reward =  0\n",
    "            done = True\n",
    "        else:\n",
    "            reward = -1\n",
    "        \n",
    "        return new_state, reward, done\n",
    "\n",
    "    def render(self, state):\n",
    "        \"\"\"Function to get the simulation of the warehouse agent system \n",
    "        \"\"\"\n",
    "        def getLoc(state, level):\n",
    "            for i in range(0,7):\n",
    "                for j in range(0,6):\n",
    "                    if (state[i,j][level] == 1):\n",
    "                        return i,j\n",
    "\n",
    "        grid = np.zeros((7,6), dtype= str)\n",
    "        player_loc = self.getLoc(state, 3)\n",
    "        goal = self.getLoc(state, 0)\n",
    "        box = self.getLoc(state, 1)\n",
    "        walls = []\n",
    "        for i in range(7):\n",
    "            for j in range(6):\n",
    "                if (state[i,j][2] == 1):\n",
    "                    walls.append((i,j))\n",
    "\n",
    "        for i in range(0,7):\n",
    "            for j in range(0,6):\n",
    "                grid[i,j] = ' '\n",
    "                \n",
    "        if player_loc:\n",
    "            grid[player_loc] = 'P' #player\n",
    "        for wall in walls:\n",
    "            grid[wall] = 'W' #wall\n",
    "        if goal:\n",
    "            grid[goal] = '+' #goal\n",
    "        if box:\n",
    "            grid[box] = 'B' #box\n",
    "        \n",
    "        return grid"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63930196",
   "metadata": {},
   "source": [
    "## Environment Details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "87fce63f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[7, 6]\n",
      "[1, 2]\n",
      "[4, 3]\n",
      "[3, 1]\n"
     ]
    }
   ],
   "source": [
    "env = WarehouseAgent()\n",
    "print(env.GRID_DIM)\n",
    "print(env.agent_position)\n",
    "print(env.box_location)\n",
    "print(env.goal_location)\n",
    "state = env.state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "dc755178",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[[0. 0. 1. 0.]\n",
      "  [0. 0. 1. 0.]\n",
      "  [0. 0. 1. 0.]\n",
      "  [0. 0. 1. 0.]\n",
      "  [0. 0. 0. 0.]\n",
      "  [0. 0. 0. 0.]]\n",
      "\n",
      " [[0. 0. 1. 0.]\n",
      "  [0. 0. 0. 0.]\n",
      "  [0. 0. 0. 1.]\n",
      "  [0. 0. 1. 0.]\n",
      "  [0. 0. 0. 0.]\n",
      "  [0. 0. 0. 0.]]\n",
      "\n",
      " [[0. 0. 1. 0.]\n",
      "  [0. 0. 0. 0.]\n",
      "  [0. 0. 0. 0.]\n",
      "  [0. 0. 1. 0.]\n",
      "  [0. 0. 1. 0.]\n",
      "  [0. 0. 1. 0.]]\n",
      "\n",
      " [[0. 0. 1. 0.]\n",
      "  [1. 0. 0. 0.]\n",
      "  [0. 0. 0. 0.]\n",
      "  [0. 0. 0. 0.]\n",
      "  [0. 0. 0. 0.]\n",
      "  [0. 0. 1. 0.]]\n",
      "\n",
      " [[0. 0. 1. 0.]\n",
      "  [0. 0. 0. 0.]\n",
      "  [0. 0. 0. 0.]\n",
      "  [0. 1. 0. 0.]\n",
      "  [0. 0. 0. 0.]\n",
      "  [0. 0. 1. 0.]]\n",
      "\n",
      " [[0. 0. 1. 0.]\n",
      "  [0. 0. 0. 0.]\n",
      "  [0. 0. 0. 0.]\n",
      "  [0. 0. 1. 0.]\n",
      "  [0. 0. 1. 0.]\n",
      "  [0. 0. 1. 0.]]\n",
      "\n",
      " [[0. 0. 1. 0.]\n",
      "  [0. 0. 1. 0.]\n",
      "  [0. 0. 1. 0.]\n",
      "  [0. 0. 1. 0.]\n",
      "  [0. 0. 0. 0.]\n",
      "  [0. 0. 0. 0.]]]\n"
     ]
    }
   ],
   "source": [
    "print(env.state)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c98d3de8",
   "metadata": {},
   "source": [
    "## Rendering State"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9021761e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['W' 'W' 'W' 'W' ' ' ' ']\n",
      " ['W' ' ' 'P' 'W' ' ' ' ']\n",
      " ['W' ' ' ' ' 'W' 'W' 'W']\n",
      " ['W' '+' ' ' ' ' ' ' 'W']\n",
      " ['W' ' ' ' ' 'B' ' ' 'W']\n",
      " ['W' ' ' ' ' 'W' 'W' 'W']\n",
      " ['W' 'W' 'W' 'W' ' ' ' ']]\n"
     ]
    }
   ],
   "source": [
    "print(env.render(state))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "39faca5b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['W' 'W' 'W' 'W' ' ' ' ']\n",
      " ['W' ' ' ' ' 'W' ' ' ' ']\n",
      " ['W' ' ' 'P' 'W' 'W' 'W']\n",
      " ['W' '+' ' ' ' ' ' ' 'W']\n",
      " ['W' ' ' ' ' 'B' ' ' 'W']\n",
      " ['W' ' ' ' ' 'W' 'W' 'W']\n",
      " ['W' 'W' 'W' 'W' ' ' ' ']]\n",
      "Reward: -1\n"
     ]
    }
   ],
   "source": [
    "# MOVE DOWN\n",
    "step = env.step(state, 1)\n",
    "reward = step[1]\n",
    "print(env.render(state))\n",
    "print('Reward:',reward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7725da51",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['W' 'W' 'W' 'W' ' ' ' ']\n",
      " ['W' ' ' ' ' 'W' ' ' ' ']\n",
      " ['W' 'P' ' ' 'W' 'W' 'W']\n",
      " ['W' '+' ' ' ' ' ' ' 'W']\n",
      " ['W' ' ' ' ' 'B' ' ' 'W']\n",
      " ['W' ' ' ' ' 'W' 'W' 'W']\n",
      " ['W' 'W' 'W' 'W' ' ' ' ']]\n",
      "Reward: -1\n"
     ]
    }
   ],
   "source": [
    "# MOVE LEFT\n",
    "step = env.step(state, 2)\n",
    "reward = step[1]\n",
    "print(env.render(state))\n",
    "print('Reward:',reward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d658f192",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['W' 'W' 'W' 'W' ' ' ' ']\n",
      " ['W' 'P' ' ' 'W' ' ' ' ']\n",
      " ['W' ' ' ' ' 'W' 'W' 'W']\n",
      " ['W' '+' ' ' ' ' ' ' 'W']\n",
      " ['W' ' ' ' ' 'B' ' ' 'W']\n",
      " ['W' ' ' ' ' 'W' 'W' 'W']\n",
      " ['W' 'W' 'W' 'W' ' ' ' ']]\n",
      "Reward: -1\n"
     ]
    }
   ],
   "source": [
    "# MOVE UP\n",
    "step = env.step(state, 0)\n",
    "reward = step[1]\n",
    "print(env.render(state))\n",
    "print('Reward:',reward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d19eda2a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['W' 'W' 'W' 'W' ' ' ' ']\n",
      " ['W' ' ' 'P' 'W' ' ' ' ']\n",
      " ['W' ' ' ' ' 'W' 'W' 'W']\n",
      " ['W' '+' ' ' ' ' ' ' 'W']\n",
      " ['W' ' ' ' ' 'B' ' ' 'W']\n",
      " ['W' ' ' ' ' 'W' 'W' 'W']\n",
      " ['W' 'W' 'W' 'W' ' ' ' ']]\n",
      "Reward: -1\n"
     ]
    }
   ],
   "source": [
    "# MOVE RIGHT\n",
    "step = env.step(state, 3)\n",
    "reward = step[1]\n",
    "print(env.render(state))\n",
    "print('Reward:',reward)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be259611",
   "metadata": {},
   "source": [
    "## Pushing the box to the goal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "520b3d34",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['W' 'W' 'W' 'W' ' ' ' ']\n",
      " ['W' ' ' ' ' 'W' ' ' ' ']\n",
      " ['W' ' ' ' ' 'W' 'W' 'W']\n",
      " ['W' 'B' ' ' ' ' ' ' 'W']\n",
      " ['W' 'P' ' ' ' ' ' ' 'W']\n",
      " ['W' ' ' ' ' 'W' 'W' 'W']\n",
      " ['W' 'W' 'W' 'W' ' ' ' ']]\n",
      "Reward: 0\n"
     ]
    }
   ],
   "source": [
    "env.step(state, 1)\n",
    "env.step(state, 1)\n",
    "env.step(state, 3)\n",
    "env.step(state, 3)\n",
    "env.step(state, 1)\n",
    "env.step(state, 2)\n",
    "env.step(state, 2)\n",
    "env.step(state, 1)\n",
    "env.step(state, 2)\n",
    "final_step = env.step(state, 0)\n",
    "reward = final_step[1]\n",
    "print(env.render(state))\n",
    "print('Reward:', reward)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:rl_ass]",
   "language": "python",
   "name": "conda-env-rl_ass-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}