01_Intro_to_environment.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.7"
    },
    "colab": {
      "name": "01 Intro to environment.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hm9T3KjCYlwk",
        "colab_type": "text"
      },
      "source": [
        "Setup\n",
        "---\n",
        "\n",
        "Make sure to select `GPU` under Runtime > Change runtime type > Hardware accelerator!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eRhzlYGRYlwq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import sys\n",
        "\n",
        "# Setup for use in Colab\n",
        "if 'google.colab' in sys.modules:\n",
        "    # Clone GitHub repository\n",
        "    !git clone https://github.com/pacm/rl-workshop.git --single-branch --branch updated-workshop\n",
        "        \n",
        "    # Install packages via pip\n",
        "    !pip install -r \"rl-workshop/colab-requirements.txt\"\n",
        "    \n",
        "    # Restart Runtime so everything takes effect\n",
        "    import os\n",
        "    os.kill(os.getpid(), 9)\n",
        "\n",
        "    # Your Runtime will crash after this - this is normal!"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SYJ-heIaiDxn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%cd rl-workshop"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PYzA28d3Ylw0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%matplotlib inline\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import os\n",
        "from PIL import Image\n",
        "from IPython.lib.pretty import pretty"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "98z39n0UYlw3",
        "colab_type": "text"
      },
      "source": [
        "The challenge environment\n",
        "---\n",
        "\n",
        "The environment for this challenge is called **`DeliveryDrones`**.\n",
        "\n",
        "After creating the environment, call `reset()` to initalize the environment."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QkHZJ1oNYlw4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from env.env import DeliveryDrones\n",
        "\n",
        "# Create environment\n",
        "env = DeliveryDrones()\n",
        "\n",
        "# Resets it and get the initial observation\n",
        "observation = env.reset()\n",
        "\n",
        "# Render in text\n",
        "print(env.render(mode='ansi'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N8wHU95VYlw6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Render as an RGB image to see things more clearly\n",
        "Image.fromarray(env.render(mode='rgb_array'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c8HdUGT9Ylw9",
        "colab_type": "text"
      },
      "source": [
        "Observations spaces\n",
        "---\n",
        "\n",
        "By default, the environment returns `ground` and `air` grids as observations."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a0O6wFz1Ylw-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Observations are returned after env.reset() or env.step() calls\n",
        "print(observation)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9_9uZ7V6YlxB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# We can inspect what's on the ground\n",
        "observation['ground'].grid"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "snCDOXVzYlxE",
        "colab_type": "text"
      },
      "source": [
        "We use **observation wrappers** to produce states that can be used with RL agents."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-V8dcm6mYlxF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from env.wrappers import CompassQTable, CompassChargeQTable, LidarCompassQTable, LidarCompassChargeQTable\n",
        "\n",
        "# Create the environment\n",
        "env = DeliveryDrones()\n",
        "\n",
        "# Use an observation wrappers\n",
        "#env = CompassQTable(env)\n",
        "env = CompassChargeQTable(env)\n",
        "#env = LidarCompassQTable(env)\n",
        "#env = LidarCompassChargeQTable(env)\n",
        "\n",
        "# Reset the environment and print inital observation\n",
        "observation = env.reset()\n",
        "print(pretty(observation))\n",
        "\n",
        "# Render as an RGB image\n",
        "Image.fromarray(env.render(mode='rgb_array'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lHfy5B0xYlxH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Print the state in a nicer way using `env.format_state`\n",
        "{drone: env.format_state(observation) for drone, observation in observation.items()}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uqyxUJAHsuMT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from env.env import Action\n",
        "\n",
        "Action??"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lQJpDbt4s0y3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "observation, reward, done, info = env.step({0: Action.STAY})\n",
        "\n",
        "print('Rewards: {}'.format(reward))\n",
        "Image.fromarray(env.render(mode='rgb_array'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CfBLPP08uKCC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "{drone: env.format_state(observation) for drone, observation in observation.items()}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c1W9xJXZYlxJ",
        "colab_type": "text"
      },
      "source": [
        "The `WindowedGridView` observation wrapper\n",
        "---\n",
        "\n",
        "This is the \"official\" wrapper for the competition!\n",
        "\n",
        "```\n",
        "Observation wrapper: (N, N, 6) numerical arrays with location of\n",
        "(0) drones         marked with                   1 / 0 otherwise\n",
        "(1) packets        marked with                   1 / 0 otherwise\n",
        "(2) dropzones      marked with                   1 / 0 otherwise\n",
        "(3) stations       marked with                   1 / 0 otherwise\n",
        "(4) drones charge  marked with   charge level 0..1 / 0 otherwise\n",
        "(5) obstacles      marked with                   1 / 0 otherwise\n",
        "Where N is the size of the window, i the number of drones\n",
        "```"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v8692JPAYlxK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from env.wrappers import WindowedGridView\n",
        "\n",
        "env = WindowedGridView(DeliveryDrones(), radius=2)\n",
        "states = env.reset()\n",
        "Image.fromarray(env.render(mode='rgb_array'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kWQA7MGLYlxN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "{drone: env.format_state(state) for drone, state in states.items()}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gUkzpIBaYlxS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "states[0][:, :, 5] # Obstacles from the perspective of drone 0"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6lIIDZmWYlxV",
        "colab_type": "text"
      },
      "source": [
        "Create and run agents\n",
        "---\n",
        "\n",
        "After creating your agents, you can run them with the `test_agents()` method"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6rAR78sQYlxV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from agents.random import RandomAgent\n",
        "\n",
        "# Create and setup the environment\n",
        "env = WindowedGridView(DeliveryDrones(), radius=3)\n",
        "states = env.reset()\n",
        "\n",
        "# Create random agents\n",
        "agents = {drone.index: RandomAgent(env) for drone in env.drones}\n",
        "agents"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K1u8t0t_sMK5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# The random agents just pick an action randomly\n",
        "RandomAgent??"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e30M9h3YYlxY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from helpers.rl_helpers import test_agents\n",
        "\n",
        "# Run agents for 1000 steps\n",
        "rewards_log = test_agents(env, agents, n_steps=1000, seed=0)\n",
        "\n",
        "# Print rewards\n",
        "for drone_index, rewards in rewards_log.items():\n",
        "    print('Drone {} rewards: {} ..'.format(drone_index, rewards[:10]))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lYpyGMMTYlxa",
        "colab_type": "text"
      },
      "source": [
        "And visualize the rewards with the helpers functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CHZoRdk6Ylxa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from helpers.rl_helpers import plot_cumulative_rewards\n",
        "\n",
        "plot_cumulative_rewards(\n",
        "    rewards_log,\n",
        "    events={'pickup': [1], 'crash': [-1]}, # Optional, default: pickup/crash ±1\n",
        "    drones_labels={0: 'My drone'}, # Optional, default: drone index \n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lWF3mrreYlxd",
        "colab_type": "text"
      },
      "source": [
        "Train a first agent\n",
        "---\n",
        "\n",
        "To train your agents, you will use the `MultiAgentTrainer()`"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ii8G6rDAYlxd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from agents.dqn import DQNAgent, DenseQNetworkFactory\n",
        "from helpers.rl_helpers import MultiAgentTrainer, plot_rolling_rewards\n",
        "\n",
        "# Create and setup the environment\n",
        "env = WindowedGridView(DeliveryDrones(), radius=3)\n",
        "env.env_params.update({'n_drones': 3, 'skyscrapers_factor': 0, 'charge_reward': 0, 'discharge': 0})\n",
        "states = env.reset()\n",
        "\n",
        "# Create random agents\n",
        "agents = {drone.index: RandomAgent(env) for drone in env.drones}\n",
        "\n",
        "# Use a DQNAgent for agent 0 - we will see how this works next\n",
        "agents[0] = DQNAgent(\n",
        "    env, DenseQNetworkFactory(env, hidden_layers=[32, 32]),\n",
        "    gamma=0.95, epsilon_start=1.0, epsilon_decay=0.999, epsilon_end=0.01,\n",
        "    memory_size=10000, batch_size=64, target_update_interval=5\n",
        ")\n",
        "\n",
        "agents"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DXh-nDjPYlxh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Create trainer\n",
        "trainer = MultiAgentTrainer(env, agents, reset_agents=True, seed=0)\n",
        "\n",
        "# Train with different grids\n",
        "trainer.train(1000)\n",
        "\n",
        "# Print rewards\n",
        "for drone_index, rewards in trainer.rewards_log.items():\n",
        "    print('Drone {} rewards: {} ..'.format(drone_index, rewards[:10]))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G9dXK8JbYlxk",
        "colab_type": "text"
      },
      "source": [
        "And visualize training with helpers functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0NBBCTvHYlxl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "plot_rolling_rewards(\n",
        "    trainer.rewards_log,\n",
        "    drones_labels={0: 'My drone'}, # Optional: specify drone names\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f7UwlPG5Ylxn",
        "colab_type": "text"
      },
      "source": [
        "Test agents\n",
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SZ0XiT3aYlxo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "rewards_log = test_agents(env, agents, n_steps=1000, seed=0)\n",
        "plot_cumulative_rewards(rewards_log, drones_labels={0: 'My drone'})"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_LUB3n6RYlxq",
        "colab_type": "text"
      },
      "source": [
        "Visualize a \"run\"\n",
        "---\n",
        "\n",
        "Share videos of your best agents! `#desiRL` `#droneRL`"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OopF87pKYlxr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from helpers.rl_helpers import render_video, ColabVideo\n",
        "\n",
        "path = os.path.join('output', 'videos', 'intro-run.mp4')\n",
        "render_video(env, agents, video_path=path, n_steps=120, fps=1, seed=None)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A65R8PA1Ylxt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ColabVideo(path)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7Yts898voWHN",
        "colab_type": "text"
      },
      "source": [
        "## Submit to AIcrowd! 🚀"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3dDLEqc1oZbV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "path = os.path.join('output', 'agents', 'first-agent.pt')\n",
        "agents[0].save(path)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eYpacPiJoo4t",
        "colab_type": "text"
      },
      "source": [
        "Download the file `output/agents/first-agent.pt` and submit it:\n",
        "\n",
        "https://www.aicrowd.com/challenges/droneracer"
      ]
    }
  ]
}