CartPole_ApproximateSolution_Batch_GD.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyNAiXOgCb5JEdQAQ2AZ1dPv",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Ogunfool/Approximate-RL-Methods-for-CartPole-Environment-on-OpenAI-gym/blob/main/CartPole_ApproximateSolution_Batch_GD.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "QS1lvLYKBadL"
      },
      "outputs": [],
      "source": [
        "# Monte Carlo Gradient Ascent (Batch GD)\n",
        "import gym\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.kernel_approximation import RBFSampler\n",
        "\n",
        "\n",
        "GAMMA = 0.99\n",
        "ALPHA = 0.1\n",
        "\n",
        "\n",
        "def epsilon_greedy(model, s, eps=0.1):\n",
        "  # we'll use epsilon-soft to ensure all states are visited\n",
        "  # what happens if you don't do this? i.e. eps=0\n",
        "  p = np.random.random()\n",
        "  if p < (1 - eps):\n",
        "    values = model.predict_all_actions(s)\n",
        "    return np.argmax(values)\n",
        "  else:\n",
        "    return model.env.action_space.sample()\n",
        "\n",
        "\n",
        "def gather_samples(env, n_episodes=10000):\n",
        "  samples = []\n",
        "  for _ in range(n_episodes):\n",
        "    s = env.reset()\n",
        "    done = False\n",
        "    truncated = False\n",
        "    while not (done or truncated):\n",
        "      a = env.action_space.sample()\n",
        "      sa = np.concatenate((s, [a]))\n",
        "      samples.append(sa)\n",
        "\n",
        "      s, r, done, truncated = env.step(a)\n",
        "  return samples\n",
        "\n",
        "def play_one_episode(env, model):\n",
        "  # Create lists for states, actions and rewards\n",
        "  states = []\n",
        "  rewards = [0]\n",
        "  actions = []\n",
        "  # Now play the game\n",
        "  s = env.reset()\n",
        "  episode_reward = 0\n",
        "  done = False\n",
        "  truncated = False\n",
        "  while not (done or truncated):\n",
        "    a = epsilon_greedy(model, s)\n",
        "    s2, r, done, truncated = env.step(a)\n",
        "    # Append to list\n",
        "    states.append(s)\n",
        "    rewards.append(r)\n",
        "    actions.append(a)\n",
        "\n",
        "    s =s2\n",
        "  # print(\"States:\", states, \"Rewards:\", rewards, \"Actions:\", actions)\n",
        "  return states, rewards, actions\n",
        "\n",
        "class Model:\n",
        "  def __init__(self, env):\n",
        "    # fit the featurizer to data\n",
        "    self.env = env\n",
        "    samples = gather_samples(env)\n",
        "    self.featurizer = RBFSampler()\n",
        "    self.featurizer.fit(samples)\n",
        "    dims = self.featurizer.n_components\n",
        "\n",
        "    # initialize linear model weights\n",
        "    self.w = np.zeros(dims)\n",
        "\n",
        "  def predict(self, s, a):\n",
        "    sa = np.concatenate((s, [a]))\n",
        "    x = self.featurizer.transform([sa])[0]\n",
        "    return x @ self.w\n",
        "\n",
        "  def predict_all_actions(self, s):\n",
        "    return [self.predict(s, a) for a in range(self.env.action_space.n)]\n",
        "\n",
        "  def grad(self, s, a):\n",
        "    sa = np.concatenate((s, [a]))\n",
        "    x = self.featurizer.transform([sa])[0]\n",
        "    return x\n",
        "\n",
        "\n",
        "def test_agent(model, env, n_episodes=20):\n",
        "  reward_per_episode = np.zeros(n_episodes)\n",
        "  for it in range(n_episodes):\n",
        "    done = False\n",
        "    truncated = False\n",
        "    episode_reward = 0\n",
        "    s = env.reset()\n",
        "    while not (done or truncated):\n",
        "      a = epsilon_greedy(model, s, eps=0)\n",
        "      s, r, done, truncated = env.step(a)\n",
        "      episode_reward += r\n",
        "    reward_per_episode[it] = episode_reward\n",
        "  return np.mean(reward_per_episode)\n",
        "\n",
        "\n",
        "def watch_agent(model, env, eps):\n",
        "  done = False\n",
        "  truncated = False\n",
        "  episode_reward = 0\n",
        "  s = env.reset()\n",
        "  while not (done or truncated):\n",
        "    a = epsilon_greedy(model, s, eps=eps)\n",
        "    s, r, done, truncated = env.step(a)\n",
        "    episode_reward += r\n",
        "  print(\"Episode reward:\", episode_reward)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "BATCH_SIZE = 64\n",
        "\n",
        "# repeat until convergence\n",
        "env = gym.make(\"CartPole-v1\")\n",
        "model = Model(env)\n",
        "reward_per_episode = []\n",
        "\n",
        "for it in range(2000):\n",
        "\n",
        "  # Some lists np.zeros(T)for batch GD\n",
        "  G_store = []\n",
        "  Values_store = []\n",
        "  g_store = []\n",
        "  episode_batch = []\n",
        "\n",
        "  for batch in range(BATCH_SIZE): # A single batch\n",
        "    # Collect states, rewards and actions for a single episode \n",
        "    states, rewards, actions = play_one_episode(env, model) \n",
        "    # print(states)\n",
        "    # print(rewards)\n",
        "    # print(actions)\n",
        "\n",
        "    G = 0\n",
        "    T = len(states)\n",
        "    episode_reward = 0\n",
        "\n",
        "    for  t in range(T-2, -1, -1):\n",
        "      s = states[t]\n",
        "      r = rewards[t+1]\n",
        "      a = actions[t]\n",
        "      G = r + GAMMA*G # Smart Update rule \n",
        "\n",
        "      # accumulate reward\n",
        "      episode_reward += r\n",
        "\n",
        "    # Accumulate G and Values for single episode for BGD\n",
        "    G_store.append(G)\n",
        "    Values_store.append(model.predict(s,a))\n",
        "    g_store.append(model.grad(s,a))\n",
        "    episode_batch.append(episode_reward)\n",
        "\n",
        "\n",
        "  # print(np.array(G_store).shape)\n",
        "  # print(np.array(Values_store).shape)\n",
        "  # print(np.array(g_store).shape)\n",
        "  # print(len(episode_batch))\n",
        "  # print(len(G_store))\n",
        "  # print(len(Values_store))\n",
        "  # print(len(g_store))\n",
        "\n",
        "\n",
        "  # Let's update Q(s,a) using approximate function\n",
        "  # update the model\n",
        "  model.w += ALPHA * np.dot((np.array(G_store) - np.array(Values_store)), np.array(g_store)) * 2/BATCH_SIZE\n",
        "  \n",
        "  # print(model.w.shape)\n",
        "\n",
        "  if (it + 1) % 50 == 0:\n",
        "    print(f\"Episode: {it + 1}, Reward: {np.mean(episode_batch)}\")\n",
        "\n",
        "  # early exit\n",
        "  if it > 20 and np.mean(reward_per_episode[-20:]) == 200:\n",
        "    print(\"Early exit\")\n",
        "    break\n",
        "  \n",
        "  reward_per_episode.append(np.mean(episode_batch))\n",
        "\n",
        "# test trained agent\n",
        "test_reward = test_agent(model, env)\n",
        "print(f\"Average test reward: {test_reward}\")\n",
        "\n",
        "plt.plot(reward_per_episode)\n",
        "plt.title(\"Reward per episode\")\n",
        "plt.show()\n",
        "\n",
        "# watch trained agent\n",
        "# env = gym.make(\"CartPole-v1\", render_mode=\"human\")\n",
        "watch_agent(model, env, eps=0)"
      ],
      "metadata": {
        "id": "PmRVcZ-W9q22",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "500b0712-09ec-444b-f843-2192fc392ad9"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/gym/core.py:317: DeprecationWarning: \u001b[33mWARN: Initializing wrapper in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
            "  deprecation(\n",
            "/usr/local/lib/python3.8/dist-packages/gym/wrappers/step_api_compatibility.py:39: DeprecationWarning: \u001b[33mWARN: Initializing environment in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
            "  deprecation(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Episode: 50, Reward: 9.0\n",
            "Episode: 100, Reward: 11.71875\n",
            "Episode: 150, Reward: 17.890625\n",
            "Episode: 200, Reward: 22.4375\n",
            "Episode: 250, Reward: 26.15625\n",
            "Episode: 300, Reward: 33.21875\n",
            "Episode: 350, Reward: 38.015625\n",
            "Episode: 400, Reward: 56.625\n",
            "Episode: 450, Reward: 56.625\n",
            "Episode: 500, Reward: 54.40625\n",
            "Episode: 550, Reward: 80.59375\n",
            "Episode: 600, Reward: 91.578125\n",
            "Episode: 650, Reward: 99.265625\n",
            "Episode: 700, Reward: 108.390625\n",
            "Episode: 750, Reward: 85.390625\n",
            "Episode: 800, Reward: 104.703125\n",
            "Episode: 850, Reward: 117.078125\n",
            "Episode: 900, Reward: 89.453125\n",
            "Episode: 950, Reward: 111.59375\n",
            "Episode: 1000, Reward: 109.265625\n",
            "Episode: 1050, Reward: 121.421875\n",
            "Episode: 1100, Reward: 161.515625\n",
            "Episode: 1150, Reward: 112.78125\n",
            "Episode: 1200, Reward: 99.0625\n",
            "Episode: 1250, Reward: 112.515625\n",
            "Episode: 1300, Reward: 97.765625\n",
            "Episode: 1350, Reward: 124.6875\n",
            "Episode: 1400, Reward: 115.015625\n",
            "Episode: 1450, Reward: 94.171875\n",
            "Episode: 1500, Reward: 125.734375\n",
            "Episode: 1550, Reward: 126.53125\n",
            "Episode: 1600, Reward: 138.046875\n",
            "Episode: 1650, Reward: 107.8125\n",
            "Episode: 1700, Reward: 153.6875\n",
            "Episode: 1750, Reward: 116.296875\n",
            "Episode: 1800, Reward: 109.296875\n",
            "Episode: 1850, Reward: 131.890625\n",
            "Episode: 1900, Reward: 141.765625\n",
            "Episode: 1950, Reward: 136.546875\n",
            "Episode: 2000, Reward: 121.796875\n",
            "Average test reward: 176.25\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5hU1fnA8e+7u/TeBKStFFEQRFwpKjYQEDRYYotRrMREE01RUVNMMAmJGhOjPxV7RY3RaGIDFEFEUJoivYj0BalLZ9n398e9s9zdnT5zp+37eZ59dubc9u7dmXfOnHvuOaKqGGOMyS156Q7AGGNM8llyN8aYHGTJ3RhjcpAld2OMyUGW3I0xJgdZcjfGmBxkyd3kLBG5WkSmpTuOWInIABFZ4sN+VUQ6J3u/JjNZcjdxEZFVIrJXRHaJyEYReVZE6qc7rlygqp+oatd0x2GymyV3k4jzVLU+0As4AbgzXYGISEF1PLYxoVhyNwlT1Y3ABzhJHgAR6Sci00Vku4h8KSJnuOVnish8z3oTReQLz/NPROR89/FoEVkhIiUislBELvCsd7WIfCoiD4rIFuAeEWkmIm+LyE4R+RzoFCpmESl0mylGich6EdkgIr/yLM/zHH+LiLwmIk0rbXudiKwGPgpxjHNFZJ57DqaLSE/PslUicqf7d20TkWdEpLa77AwRWetZ9w4RWeeehyUiMtAtryUif3fjX+8+ruXZ7jb371ovItdWiq2WiNwvIqtFpFhEHhOROqHOl8lCqmo/9hPzD7AKGOQ+bgvMB/7hPm8DbAGG4VQgznaftwDqAPuA5kANoBhYBzRwl+0Fmrn7uRg40t3HpcBuoLW77GqgFPgpUOBu+wrwGlAPOM7d77QQ8RcCCox31+8BbPb8TbcAM9y/rRbwODC+0rbPu9vWCbL/E4BNQF8gHxjpnrNanvP3NdAOaAp8CtzrLjsDWOs+7gqsAY70HLuT+/gPboxHuOd2OjDGXTbUPbfHuTG+7Mbc2V3+IPC2e+wGwH+BP6f7dWU/SXyPpjsA+8nOHzc57QJK3KTxIdDYXXYH8EKl9T8ARrqPPwEuBPoBE9yEPBQ4E/gqzDHnASPcx1cDqz3L8oGDwDGesj9Fkdy96/8VeMp9vAgY6FnW2t1/gWfbjmFifTSQaD1lS4DTPefvRs+yYcAK97E3uXd2PyQGATUq7W8FMMzzfAiwyn38NDDWs+zoQHIHBOeDspNneX/gm3S/ruwneT/WVmgScb6qThKR03Fqhs2B7UAH4GIROc+zbg1gsvt4Cm4Ccx9vA04H9rvPARCRq4Bf4CRTgPruMQLWeB63wEm83rJvo/gbKq/fw33cAXhTRMo8yw8BLUNsW1kHYKSI/NRTVhPnm0ioY3uXAaCqy0XkVuAeoLuIfAD8QlXXu+t/G2IfRwKzKy0LaAHUBWaLSKBMcD4gTY6wNneTMFWdAjwL3O8WrcGpuTf2/NRT1bHu8kByP819PAUnuZ/uPkZEOgBPADfjNNM0xmnGKM9GODXRgM04zTTtPGXtowi/8vrrPX/DOZX+htqqui7E8StbA/yx0vZ1VXV8FMeuQFVfVtVTcT4wFPiLu2i9WxZsHxuC7D/gO5zmr+6e2Bqpc3Hc5AhL7iZZ/g6cLSLHAy8C54nIEBHJF5Ha7kXCtu6603HakvsAn6vqApwk1ReY6q5TDyeRbQYQkWtw2o+DUtVDwBs4F1brikg3nHbuSH7jrt8duAZ41S1/DPij+yGDiLQQkRHRnQrA+WC6UUT6iqOeiAwXkQaedW4Skbbuhdq7PccuJyJdReQs90LpPpykHPg2MR74tRtbc+C3OOcenKauq0Wkm4jUBX4X2KeqlrnxPSgiR7jHaSMiQ2L4+0yGs+RukkJVN+NcYPytqq4BRgB34STnNcBtuK83Vd0NzAEWqOoBdxefAd+q6iZ3nYXAA255MU5zyacRwrgZp+lmI843iWeiCH0KsBznmsH9qjrBLf8HzgXHCSJSgnPhsm8U+8ONfxZwA/AwTrPTcpzrBF4v41xzWInTfn5vkF3VAsbi1LY34lw8DXQ5vReYBXyFc0F7TmAfqvoezgfuR+6xK/foucMtnyEiO4FJOB+4JkeIqk3WYaofESkEvsG5SFmahuOvAq5X1UmpPrapHqzmbowxOciSuzHG5CBrljHGmBxkNXdjjMlBGXETU/PmzbWwsDDdYRhjTFaZPXv2d6raItiyjEjuhYWFzJo1K91hGGNMVhGRkHdhW7OMMcbkIEvuxhiTgyy5G2NMDrLkbowxOciSuzHG5CBL7sYYk4MsuRtjTA6y5G6MyWhvzVtHyb6D6Q4j61hyN8ZkrAXrd3DLK/MY/cb8dIeSdSy5G2My1t4DhwDYuGNfmiPJPpbcjTEZz0avjV3E5C4i7URksogsFJEFInKLW95URCaKyDL3dxO3XETkIRFZLiJfiUhvv/8IY4wxFUVTcy8Ffqmq3YB+OJP6dgNGAx+qahec+SdHu+ufA3Rxf0YBjyY9amNMtSIi6Q4h60RM7qq6QVXnuI9LgEVAG5wJkJ9zV3sOON99PAJ4Xh0zgMYi0jrpkRtjqg1rloldTG3u7qTCJwAzgZaqusFdtBFo6T5ugzPbfcBat8wYY0yKRJ3cRaQ+8G/gVlXd6V2mzsdqTB+tIjJKRGaJyKzNmzfHsqkxxpgIokruIlIDJ7G/pKpvuMXFgeYW9/cmt3wd0M6zeVu3rAJVHaeqRapa1KJF0IlEjDHVnDW1xy+a3jICPAUsUtW/eRa9DYx0H48E3vKUX+X2mukH7PA03xhjTNSsqT1+0UyzdwpwJTBfROa5ZXcBY4HXROQ64FvgEnfZu8AwYDmwB7gmqREbY4yJKGJyV9VpQKgvRwODrK/ATQnGZYwx1iyTALtD1RhjkmzDjr1c/Nh0tu0+kLYYLLkbY0ySPT5lJV+s2sabc6v0JUkZS+7GmLQq3rmP9dv3hl0nW6+rpjPuaC6oGmOMb/r+6UMAVo0dnuZIkmdpcUm6Q7CauzEm82XbddXpK7akOwRL7saYzJetzTLhLN64kw8WbPRt/9YsY4wxaTD0758A/jVHWc3dmBSYs3obkxdviryiMUliNXdjUuDC/5sO5NZFw9TIttb2zGE1d2OMyUGW3I0xJgdZcjfGmBxkyd0Yk/Fs6N/YWXI3xpgcZMndGJPxsnXo33RO7G3J3RiT8axZJnbRTLP3tIhsEpGvPWWvisg892dVYIYmESkUkb2eZY/5Gbwx2WbRhp2RVzImCaKpuT8LDPUWqOqlqtpLVXvhTJz9hmfxisAyVb0xeaEak/1+/OLsdIdgqoloptmbKiKFwZa5k2dfApyV3LCMyU3WuhCbbG1rj+T12Wt9P0aibe4DgGJVXeYpO0pE5orIFBEZkOD+jTEm5/zqX1/6foxEx5a5HBjveb4BaK+qW0TkROA/ItJdVas0NIrIKGAUQPv27RMMwxiTy5Zv2pXuELJO3DV3ESkALgReDZSp6n5V3eI+ng2sAI4Otr2qjlPVIlUtatGiRbxhGGOqgV37S9MdQtyem76KFZtT/+GUSM19ELBYVcsbj0SkBbBVVQ+JSEegC7AywRiNMcZ3m0v2U79WAXVq5se9j/lrd/D67DUVyn739gIa1C5g/j1DEg0xJtF0hRwPfAZ0FZG1InKdu+gyKjbJAJwGfOV2jXwduFFVtyYzYGOM8cNJf5zEZU/MiHv7uau3cd7D03jus2/LywL980v2lbJxxz6GP/QJm3buq7CdXzc6RdNb5vIQ5VcHKfs3TtdIY4xJugcnLuXjpZt566ZTfNn/l2u2x73tqi27q5Spp3/UCzNWsWD9Tl6bVbFmX6aQ70OvIJuswxiTNf7x4bLIK2WQaCrlTs09+dndhh8wxpgUCJXoy3y6+cGSuzEGgH0HD/HwR8s4UFqW7lCSYsbKLfzqX1+mbPCuYIfxFm3bczDodmU+xWfJ3RgDwONTVnL/hKW8NPPbyCunQaxJ+rJxM1JyJ2g43pDHf746pce25G5MGjz5yUomL96U7jAq2HPQ6Uu+72Bm1tzjbb5I54iSGmTAifsnLK3w/JBP7TKW3I1Jg3vfWcQ1z36R7jAynvcyYzQ19y/XbGfH3uDNH+H41XSTzg8WS+7GmIzlzY3RVHBHPPIpVz01M+Q+Qnnv640xxZVMfuV/S+4mrbbuPsDBQ5nZDJCo12evZdYqu4cvWYI1cVRY7laTv1y7I+Z9B+ujHqt4a+l+fWuw5G7SqveYidz6yrx0h+GLX/3rS77/2Gdxbbtz30F+8MQM1m7bk+SoskvFZpnw64ZaHk3y9Kv5xKbZM9XaO/M3pDuElNmwY19Ub/j35m9g+ootPJRlN+0k29pte8sfx10zTlIsXks2ljDi4WnsjjCg2eNTIw+tZc0yxuSAA6VlPPPpqnSHkTV+On5u+eNI/cETSZLRfOD+/r8LKBz9DgBj31vEl2t3MPObLWGPX7Iv8miWflXubfgBY3zwwYLQF+hmrNwSclk6LFy/k0Z1a6Q7jIgi5cBQCTpZyTPbPpSt5m6MD370QvbMlTrsoU84ZexH6Q4jIn9r7ocfp3zsdRt+wOSadF5synSxnJqhf5/K7a8nb9q2SL1SYnHwUBnPf7aK0iT0iIq/zT2KC6qexxc9Oh2AN+euZfKS4DeaBfrSX/vsLP7pXheJ9/WczPPtZcndGJ9VftMne9LnxRtLeG1Wem+zD+W56av47VsLeGFG4kMahEqeFz82nYc/WhYy+e8PM1bOI5OX84vX5lXYds+BQwD8/NUvueaZqjeanfbXycxZfXho4AcmLq2yTiyW+TSFoCV3kzbVpeJeXf7OYAI13GguLO7cF/7O0lDn8YtV27h/wlJ+8tKcoMt73jMh5D7v+2AJb8xZF1PtefXW4N1TJc5P7T+9uyiu7SKJZiamp0Vkk4h87Sm7R0TWicg892eYZ9mdIrJcRJaISGrnlTImA1Xj3B6TnvdMYGlxCYWj3wk6aUakNvdJi4rjPrZ31wdKy9jr1t5j20d8/+l0Dvn7LDA0SPmDqtrL/XkXQES64Uy/193d5v9EJP4JCU1OS3XS21yyn8LR7/BumvvVf7Yis3rLVCZJmjji7S/Xs7S4BIj+28vgB6cC8N8v11dZ5h0zZvqK72KO50BpGfNCzLRUObw73/gq5v2H6yEVTlm6Bg5T1alAtPdQjwBeUdX9qvoNsBzok0B8xiTNko1OonkxCe2/sahco9sZRRNFQLISbTr8bPxcPlgQvjY9+9voh2c464Ep5Y9/8MTMMGtWVVamjPnfQs5/5FNWRtEbZnkcPWYmLYpvlM9MHM/9ZhH5ym22aeKWtQG8EwSudcuMyRjxNI3u2l/KX95fzPTlsdcYE+qilyONOuu272FHkMkqLno0+PAMyf6rV2/dw/x1zpgz2/ceZHPJfn7x6uFhL9Zv31th/a/X7Yxp/8WVJr2ORabNxPQo0AnoBWwAHoh1ByIySkRmiciszZs3xxmGyWap7gqZSKI89S8f8ejHK/jBkxVrjF+v28FrX6wJsZV73DTm57fmrWNNiAuAqfTarLUM/NuUyCu6kn3OFKdZLqDvnybxxtx15c8TndSj758+jHvbjBo4TFWLVfWQqpYBT3C46WUd0M6zalu3LNg+xqlqkaoWtWjRIp4wjKlAVZmwYGPEPtXxNHVsDzFF2rn/nMbt/469fTZVbnllHuc/8mnS97tz30H2l8Z20fG7XU5yffKTleVNZKmyYtMu1rm18zwR32rL8cioZhkRae15egEQ6EnzNnCZiNQSkaOALsDniYVoclWyX9ITFhYz6oXZPDZlRfDjpegN/Zv/fF3hebqbVrbsPhDT+tHE2/OeCTG3ewfc+84ihj/0SVzbxmvB+sPNLPFcxXjfx/He/ZqJKeLYMiIyHjgDaC4ia4HfAWeISC+c9+cq4EcAqrpARF4DFgKlwE2qGnufImPiEKgZrtsevP0z8BZK9k1ElVW+YSeRD5VMvqA6+9ttcW9bmuKq84OTErvR6MYX/RtOwq9KRzS9ZS5X1daqWkNV26rqU6p6par2UNWeqvo9Vd3gWf+PqtpJVbuq6nv+hG1yQXW+uSdar85aE/HmnlCS3Za7bvte9hyo2tNn+vLvuO+DxUk91tOffsPGHfFfpAwn1g/3A2HucE2GjGqWMSYbpWssm0QPG+mCbTzHXbB+B1+vCz5j0V/fX8KTn1Qdh/yUsR/R7bcfMLPSqJY/eHImj0wO3hRWNaboT8aUpfFPIB7oXx9MrN+GLn9iRtxxRCPTessYk3GizRvx3iYejXhuSFm/fS/LK40vEs1e/vzuIp799JuQy8PtY/hD0zj3n9NCjoB47zuhb4m/dFz8yS6WD7pExhoL3AwVTKz//kSan6LhV83dxnM3aZOqC41/eX8xJfsOMujYluVlBw+VMer5Wfzi7K70aNsoaccK1pYc6e882R1ud9XY4TEdK9IsP9HUkgc+EH33xGSI5T9+KIr4v8iBOWrTdoeqMdlm/OerKzx/9OMVvDhjdYUeE8s37WLyks386l/JGyoXql5MhfC11WnLgt8UlYzvFrGmjBc+S+zO3WBNOZXF0iwzeXHkZplxUUxjV1mqL+ZGYs0yJuf42QSuqrw5d22FAaDu+2AJUDFxJruFJtiAV+H+zB8+Fbw7YaDnT8A33+1m0sJi3vlqA3sOlLIsTJty+XFjPL974hgsy+vedxZF7NYXS0iVz0GVfakycWHsg4Xd8PysmLfxkzXLGBODz1Zu4eevfsmsvlXbS0XgnH9E18964sJiurZsENW6yzeV8HaQAa/icf+Ew133JiwsrtAGfmrn5kwLMwzCuu17WbN1Dye0b1xe9pOXZvPw5b0p2VcaVXNHvMpUyQ/zvSOWQ3+1NvgF33j25eW9UzUTWHI3WeeZT7+hf6dmHNOqYcqPvcsdnKt4Z9U3ciyV9Ruen0Wtgui+4N75xvyg5Yn20vn8m4rtyuESO8CZ93/MgdIyFo85PJjru/M3suf7hzj+D6HHNq9swfoddGvdMKYL0JGnwkteIsusxpX4WbOMyTq//+9Chv49tXciVlX1nTN5SWxjGYWbySe+CPwVql92XoxNUMMfmsZrs2LrhlnmHto7PK/Xh3GOnBjMwvWxDe6VqazmbkwM/OzuGKtY3ru/e+trBnRJzlhLlY8brntjKIsjjAEzZ3XFZq9Ak8+ljwcf7XFFEqeU+8Vr8yKvlAX86i1jyd34Yvf+yGOW+3lB9YEJzsXTSLXueD4EFqzfQUFeHl1bRdcWH4vnPvuW5xLotfKjFw5fLKzcBPLyzNWVV48o0v/o+emrKjwP1EJDfSh481iizVV+zT2aamkbfsCYeDzkzggfryufmslT00LfoBNJILnsiuJDJlbDH5rGkL+HvkmmsptfDj63px+8k2Ok4obcyh+Oj4cYtC3AOxXeb99a4EtM2cavC9yW3I0vommnDndx7ZNl3zHmfwud9VR58pOVcfVymLs6+LRqyTZu6gq+WBX8TsZPQvRl91syUkak2vWbcyuO6P3I5BUUjn4n5PrzPUMeBLsnoDqysWVM1lpaXJJQu+LijSXc+84ifjZ+bpVl+0sPsS3GIW29QjXKLC0uiWm/f3o3uQNnJUMyxtIJDBecrnF5qgPrLWOy1uAHp/LI5OVVyr8riS55lh5yXv0l+6v2wLju2VmcMGYikNxeKYMfnMq5/5yWxD2mXjLOx/++cgZ8tdzun4yaicmYSCq/YOcGuXPzfveiZyICfb6Xb9rFB3FMqLB22+Ep6Drf9S7fe/hwQl9XaV7NyibFcXdkKiUzZ1hu94/V3E1WC9b8kczX9KC/TYl4c08wO/cdvuBaWqYR74r0uj7DbmOv8u0oSSe4cPQ7zF3t78iI1ZlfMzFFTO4i8rSIbBKRrz1l94nIYhH5SkTeFJHGbnmhiOwVkXnuz2O+RG1yQqxfR1PVNDA9jg+JTBAYOycgmXeD/n1SYr2fTOpFU3N/FhhaqWwicJyq9gSWAnd6lq1Q1V7uz43JCdNku2z6Wh+sCSkbJfPDMJ5vRSa9oplmbyqwtVLZBFUNfJ+dAbT1ITaTxaLJK7Hmngy66dQX4boQxmPjTn+mqTPZIRlt7tcC3rlSjxKRuSIyRUQGJGH/JgtFVWuMYp0dew7y6QqrNcYj2pEvTW5KaPgBEbkbKAVecos2AO1VdYuInAj8R0S6q2qVEX5EZBQwCqB9+/aJhGEyUKj23sEPTqFN4zo8c02foOts232gvGsjOKMyfp4Ds+0Yk2px19xF5GrgXOAKda+Mqep+Vd3iPp4NrACODra9qo5T1SJVLWrRIjkDJZn0e3HGt1zy+GdVau4zVm5hz4FSlhbvCjsq45JKk1B4n/t5QdXbY2Hq0thGjTQmE8VVcxeRocDtwOmqusdT3gLYqqqHRKQj0AWIfR4sk7V+/R+nU1XnI+pXKN9z4BC3vf5VhbKyMu/yUhZvLKky+04q2tnfmreOW145PMLgzG/sm4LJfhGTu4iMB84AmovIWuB3OL1jagET3YGDZrg9Y04D/iAiB4Ey4EZVtXdKjlNVbnh+Nlf0O9y8NifIjPFLKo0U6G2WWbh+J99/rOowsam4hhq4C9OYdDixQxNf9hsxuavq5UGKnwqx7r+BfycalMk+kxYVVxjxL9I44NH2DPE2l/hVi8/xTjhZqXf7xsxJ0aBv6XZh7za+7NfuUDUJi7YtPJ4xNFIx6Uaud7HMRiN6+ZPwkun5a/vQsmGtdIcRkiV3k5CVm3fR8a53o1o31NRr0dq4w+m3vWRjSVInOZ682C6gZppsGIXyyMa1eeumU9MdRkiW3E1CPloc/ZyY3+0KPQpkSRSTany36wAvz1zNkL9P5awHPo76uJEcOJTYHKkm+TI/tTvfKls1qh3Xtse3bXR4Pz41DFpyNxnhmme+iGq9u96cD0DJvuTPsGQyR8uG8SXNVOrYvB4APdo0irAm3Daka4XnF/b2/6Z+S+7GmIzTvH7mtmUD1MzPK78edFwUyf2mMztXeJ6KZidL7iajZUPbq0m+7Pq/xx5rKv46S+4mIVn1HjTVxud3DfT3AJ5m8kjvgYK8qm3qqXjfWHI3xmSk//00/p4oR/jcZu9N1+ES9Qe3nsb8e4YA8PAPTigv73NUU58iO8ySu8laY9/LvEmpDVyUhIuFCrRvVjfxYGJ0y8Au5Y+Pci+YRhJuUpS6NfOpUzMfgDaN6wBwfLvGHNemEZcWtQP8u8/CkrvJWo9NWZHuEEwQNQsSy1YDujSnyKdb8sO5bUjXCsn9hgEdq6xz97Bjq5TF3MTibuD3zXOW3I0xSfPy9X1DLhvWoxVPjSyKuI8/nt+Dgvw8ahXEnp4a1C6okKC9uh/ZMOy2Z3drSZ6nfTzYsACXBKltZ+plJ0vuJqPttP7sWaNji3qc3Lk5oUbrEZGYaqu1CvK58ITYhiF49IoT+fnZVUcZ/+zOszjv+CPDbnt0ywblj9s1rUPtGvlc0ffwYHjHtGoQ9E8L9WESknsSfjawC6d2bs7wnq1j2z5KCU3WYaqnA6VlvDTzW67s1yHdoZgMkh8hc+eJRHU3pnc3TerVTDQsAFo3qhN188k7PzuV1o3qVIhlzIjuXNm/sHwIDe/f0a5p1WsDbZvUYe22vWGPc2TjOrwY5ptOoqzmbmL25LSV/P6/Cxn/xZqwF5NMbC4pyu6piPODdPnzyhM4tnXoppHABUe/BF6r5xzXqrzsLxf1AKBx3RrlZd2PbETTEB8qibSTd3LnOLi50g1NfrHkbmK2Y49Te9mzv9T6uSfRH0YcF9P6p3RuVuF5YQK9Sz765elxbxsQKbkL0KpRbVaNHc77tw7gb5ccH3w9qbhNLKJJvu2b1eWF6/oAMOjYlkz8+Wl8+Ivwf3/gZR7YfTxJvmHtGqwaO5yzu7WMfeM4WHI3MStzM3qejZWbVLVr5Edcp06NfE4/uuq0lBec0IbHrwx/sXLub84OuaxNk8O15no1I8cRTKTk7nVMq4ZVZuvS8l4k8b+uwlU26rjnt17NAgZ0acGqscNpVr8WXVo2oFmI4Q7aNnE+MFu4y1MxBHWyRJXcReRpEdkkIl97ypqKyEQRWeb+buKWi4g8JCLLReQrEentV/AmPQLzZ+TF8GaurmoV5PH6jf0rlP0yyAW/SJ65+iTAacsN1ovkwUt70bVVgwplx7Wp2AQSrv26VsHhhB7vh/Y93+setDzwjaJyYqzr+RBZPGZozA18932/Z0zrX9G3A7cN6cqo06p2cQzlhgEdeeKqIoZ6mnKyRbQ192eBoZXKRgMfqmoX4EP3OcA5OHOndgFGAY8mHqZJh6XFJezcV3UM9sDsSGP+tzDVIWWlosKKdyOOPKUw5LrtmlZtdz63Z2tO6dycfh2bMvaiwwktUpPYg5f0iiq+1pWHrY0it78yql/542l3nMnHvzqD3u2r9k1fNXY4Pz2rS9Dddj7i8IdRfp5wvtszplGdGkTjYrdbole4z6WaBXncdGbnqL4heeM6u1vLKh9M2VCtiSq5q+pUoPJcqCOA59zHzwHne8qfV8cMoLGI+NPXx/jm82+2MvjBqVz6+Iwqy7JrUCf/NKkbXRKqrHJi8NZgP7j1tCrrP/yD3tQsyOOVUf1jmm9TBB64OHi7ttd7twyIep8B/Toebu9v26QuhWHu5ix/tQTJiIGmHAFuG9yVBb8fQv1ahzvxRfoSMeW2M3jiqsh955Ml2tf+UyNP4of92vt+kTicRNrcW6pqYGbhjUDgKkEbYI1nvbVumckilzzuTFa9aMPOKsu8c1tW5zQ/97eDuaSoLef2bM2TIRJM5fNTuUfMv3/cn3d+dvjiYt2akXsnn9zJSawdmjkJ9ar+h7ukBnp/BFx0YvgeOMe0akDjuhWbawT41eCjw/ZsiUYf9xtLuITYvL5zbMVp5qtXK7re2Y/8wGnt7dCsHmd2rXoNwm/B2t4b1D4ce9dWDbj3/B5pbbpMSj93VVURiel9LiKjcJptaN++fYS1TabYd/AQ89ftSHcYSTWi15G8NW99XNv+9ftOUvZO5B1O5UR6YgcnAUY7jgnAyJMLGXpcaz5YsLHKsktPas/jU1eycvNuAr1XTsYAABfOSURBVFXlt28+JeTFzlCDc918VhdWb90T9MM9kiMaOBcfrzq54n0Qwfq4v37jyXy2cgs18sPXM9s3rcvqrXsA6NCsboUbfwry8zi5UzOmr9gSc6zJMuPOgdSpkc/xf5iQthgqS6TmXhxobnF/B+ZbWwd4G8PaumUVqOo4VS1S1aIWLVL/yWsq2nfwEMf85j3e/3pDlWV/+O/htvWDlaake+Sj5b7H5jdV+OT2MxPaR6gKWs0ISSsekaZ3a+J+gNTId4Lq2bYx3Y8MPqFEQZj4itwPnj9eELyL5ss39OWxH55YpfymMzvzwMXHM7yHk4DDfey1a1q3/Jb+cLx3igb7IhAYi8Y7YfWC3w+JuN9YhfpbWjWqTaM4m+n8ksgr721gpPt4JPCWp/wqt9dMP2CHp/nGZKh12/ey72AZf31/SZVlT3/6Dc98+g1Q9etoNHOfZoN2TeuWf9WPR+XzMuhYp5XyjqEVp1dT1fJ14+1yGMmjP+zNmPOPK2+2CcXbnONV0+2Nc3FRW6bdcSZX9O3Ar4dXHTDr5E7Ng/YiqVmQx0Unti3/OwPfSnq2jTxjUTinBekCGnDLoKP58JenV7hIG20TTyzq1yygX8emFYbv9Zpx50Cm3ZFYRSFZou0KOR74DOgqImtF5DpgLHC2iCwDBrnPAd4FVgLLgSeAnyQ9apM0d785n8LR77Dqu91h1/u9W3vPxYupgckUhvdsHfEC2Px7Bke9z1Vjh3Nl/8KY47ltSNfy7pIN4khQRzSoHXFoiH/d2L/KTVMj3WT/zNXODT4iUt7Pe+TJhYwZEbyrYyQnFTZl4s9PC/lhEq1wx8/PEzq1qB9yebLk5QmvjOrPGV2PCLq8VaPa5ecs3aLtLXO5qrZW1Rqq2lZVn1LVLao6UFW7qOogVd3qrquqepOqdlLVHqo6y98/wSTipZmrAbjuucj/pvGfr+aC/5vud0gp9+tzu5U/Pvf48B276tcqoFe7xvzGs000bndr8NF8Nt50ZmeucJNzQb4/F+ROKqw6WcSvz+3G1NvOpEeQGnaN/Ly4PqgCurRsENcNQN5t2japy9Durfjn5cFrzaYiGzjMRO3ON+anOwRfeMcRuX3IMbSoX4t731kUdF0R4T83nRJxn7cMqjhSYMPaTntsvVoF5ZcVE202SPa3qBr5eREnyOjfsRmfrUzPhcv8POGxK6u28ZvgLLmbigS27j6Q7ijSJj9PnLbdIMk9XJsvwFMji8q/AVXuRnjZSe3Yvb+UkScXUrtGPr8efiwDj03NGCOVBXqzxOP56/pwoLQs8opJlnuNgf6z5G4qWLl5N73HTEx3GBll7IU9OO3oFhwZoT0+XLIuyM/jR6d3Kn9+fZBZfrxiqZXH0tgx486B1K0V/4XcGvl5EbstJlOijVI18/M4cCj1H0aZwJK7MZU0qdQX/bI+6bsPI1w7dWBuzroxNO+E60KZiz6+7QzWbQ8/rnquslEhTUpcHOFOyUzSokEtZtw5MN1hAKH7z4MzEfVtQ7rys7NinAmoGjmycZ2gF4+rA0vuxndXn1zIlQl2g/Nq17QOd55zDABPX13EYB/Gx463hvvMNSfx8a/OSPj4TevV5MbTO/HyDf1CrlOQ7wyEVcen/vKZJAd74PrOkrvx3YAuzenZtjGTk5D0AHq1a8Ko0zqy5N6hnHVMS8ZdVUTbJukboMnrzK5HhB1EK1oiwuhzjqkwr6cxsbDkbnwXuNAYy/gpofzz8hP484U9EJEKY5BPve3MCk0p3lmKjmnVIOQYKsbkKkvuJqWev7ZPzNtc2PvwoKLnHX9khSFhA/LypEL7tPdr/OBuLTmuTWK3vps0yYaB0zOUJXeTUt5xwEPpc1TFC2CBKc6CTS8XireTSeXm2p+dlZoJik3iOjR1vu21yZBmt2xiXSFNStUsyKNd0zqs2Rq6e1r/js34/JvDc8P0atcYgB/0TaxL4iuj+tG6UW06NKvHQ1GMZukdZtakx+V92lHYvC79o6gUmIosuZuMMvOugRwoLeMfHy4rLzunR2s++uXpdIxhYKjWjQ7X9AZ3c0Yu9H5rWDV2OIWj3wm7j7dvPoXinfujPqZJPhHh5E7N0x1GVrLkbnwVbJTFujVCv+xaNqzN2m2Ha8uB7WNJ7OAM1HXWMUdwznGt4p6xvnHdmlUm1zAmW1ibezV29K/f8/0YHYIMRPXkyPBzXgaS8ZGNavPp6LOiPlaz+rUY0KU5Y0Z054gGtRnWo3Xcid2YbGfJvZratb80LQNAgTMxxj8u6xVyeW13sohOR8RWW8/PE164rm9MQ9N2jvEYxmQLa5apZiYv3sRDHy3j+lPDD1zltxG92lCvZgHXP191HPlm9WvxwnV9ON69kOqXKbedQZN61uxiclPcNXcR6Soi8zw/O0XkVhG5R0TWecqHJTNgk5ifvTKXuau388+PlkVeOQGXRjEvZjgDurQoHwPdLx2a1fP9GMakS9zJXVWXqGovVe0FnAjsAd50Fz8YWKaq7yYjUBO/A6VlfLfL6fWR57ZBL95Y4usxv9fryIjrWHO4Mf5JVpv7QGCFqn6bpP2ZJLr55TkU3TsJCD/KYKJurzQZtDEmfZLV5n4ZMN7z/GYRuQqYBfxSVbcl6TgmDhMWFpc/zvcpux/buiE/OaMznVrUp3f7Jiwtjv6bQc+2jdiwYx+ndLIbVYxJloSTu4jUBL4H3OkWPQqMwbnrewzwAHBtkO1GAaMA2rdP32QI1Ymq+tI18O5hxzK8pzOx9JDuzg1DsST3ZvVq8vbNNrCXMcmUjJr7OcAcVS0GCPwGEJEngP8F20hVxwHjAIqKimy05hQ4VKZJaZY5tXNzTj+6BcU791GmcMNp6e15Y4ypKhnJ/XI8TTIi0lpVN7hPLwC+TsIxTBJMXrI5odvp7/t+T3bvL+XqU46KuK5NrmBMeiWU3EWkHnA28CNP8V9FpBdOs8yqSstMigx84GNOKmzKvecfV152Q5A+5cE0qF1Ayb7SCmX/vflUerRN7rC5ee7XCL+uAxhTnSWU3FV1N9CsUtmVCUVkkmLF5t2s2Lyb809oE3nlSsrKqla7mzeI7WafaJr2B3RuzjWnFPLjMzrFtG9jTGQ2/EAOemTy4eFsLxs3I+rthvVwLoaWBknufjSzFOTn8bvznHFgjDHJZck9Bz368Yq4tvvNud0AKky4XLMgj7O7taRlQ0vAxmQTS+45YPmmXdzyylz+NWtNQvtp3agOdw87ltdvPJl+HZ3ZkHq1a8wTVxVZu7gxWcYGDssBg/42BYC35q2nz1FNE5p2MtCt8Z+X9+akP07i+ye2jWs/7Zo4Q/2e0TX6qfGMMcljyT3HnH7fx0nZT4sGtVg1dnjc27dvVpdZvx5EMxt10Zi0sORufNPcndjaGJN61uaexbbtPhBxHtBoXRBHl0ljTOay5J7FNpXEf7fp9adWvMu0UR0b19yYXGLJPYvV9XRZjEW7pnW4fegxFcrUxgswJqdYcs9i8ebjod1bUbPA/vXG5DJ7h2eR3ftLK9SwV2zelbR9W73dmNxiyT1LrN22h+6/+4AXZxye7OqaZ7+Ia1/BxnS3Vhljcosl9yyxeuseAO77YAl3vvFV0MG9wvnrRT0ZVWnc9a9/P4Q3fnIyAGceYzcbGZNLLLlnicDE1jv3lTL+8zVs3hVbT5kDh8poUte5oShQb69fq8CZEu/eczjrmJbJDNcYk2aW3LNEXqWmlGjHkbnYHT7g4KGykOvYxVVjco+9q7NE5XG77p+wNOI2K/80jKv6FwJw+tEtGN6jNSLEPV6MMSZ7JGOC7FVACXAIKFXVIhFpCrwKFOLMxnSJqm5L9FjV1Vvz1sU8jO+FvduQlyf0aNuowhgx3/w5/vFijDHZI1ljy5ypqt95no8GPlTVsSIy2n1+R5KOVe3c8sq8mNZPZMAvY0xu8KtZZgTwnPv4OeB8n45jKhl0rF0YNcYkJ7krMEFEZovIKLespapucB9vBCzjpMi4K09MdwjGmAyQjGaZU1V1nYgcAUwUkcXehaqqIlKlU7b7QTAKoH379kkIw9w97FjybMYkYwxJqLmr6jr39ybgTaAPUCwirQHc35uCbDdOVYtUtahFC7uBprJXPl8d80XU2nEOJGaMyT0J1dxFpB6Qp6ol7uPBwB+At4GRwFj391uJBlqdHCpTRr8xH3B6vUTys4FdqJEnXHZSO79DM8ZkiUSbZVoCb7pjlRQAL6vq+yLyBfCaiFwHfAtckuBxqpXJiw9/0en7pw8jrl+rII+bzuzsZ0jGmCyTUHJX1ZXA8UHKtwADE9l3dRaotUcr3nHdjTG5y+ZQzSBlZcrHSzfxXZTjxlx7ylG0alSLH/br4HNkxphsY8k9Azw17Rt27DnAQx8tj2m7O87pSq0Cq7UbY6qy5J4BxvxvYVzbWWI3xoRiyT3NJi4sjnrdq08u5OeDjubtL9fRtmldH6MyxmQ7S+5pdsPzs6Je98ChMhrVrcGV7kiPxhgTig35m+EGd3NGbji+XWNuHdQlzdEYY7KF1dzTSCNMXPrjMzrx07M689XaHfTr2CxFURljcoEl9zTYVLKPOjXy+fV/vg673lHN61G3ZoEldmNMzCy5p9COvQd5+8v1/CZCUv/e8UeyaMNOhnRrlaLIjDG5xpJ7Co18+nPmrdkedh2baMMYkwx2QTVFJi0sjpjYf3detxRFY4zJdZbcffLfL9ezdtseAB6bsoLro+jyeEVfG0bAGJMc1izjg0Nlyk/Hz6VN4zrcf/HxjH1vcch1WzasRfFOZyyZmgX2WWuMSQ7LJj7Yc6AUgHXb93L5EzOCrvPy9X25bUhXZt41KJWhGWOqCau5+2D3/kNhl1/VvwP9OzXj5M7NAfjHZb3Yf7AsFaEZY6oJS+5JtGbrHlZv3cMVT84Muc7KPw2rMs/piF6RZ1syxphYxJ3cRaQd8DzObEwKjFPVf4jIPcANwGZ31btU9d1EA810haPfCbv81M7NefzKE20Ca2NMSiRScy8Ffqmqc0SkATBbRCa6yx5U1fsTDy/zFe/cF3EqvCZ1a/Di9X1TFJExxiSQ3FV1A7DBfVwiIouAatW+cPBQWcTE/ty1fTjuyIYpisgYYxxJaXMXkULgBGAmcApws4hcBczCqd1vS8ZxMkVZmXLz+Dm8O39j0OU/PqMTfY5qSoemdenYon6KozPGmCQkdxGpD/wbuFVVd4rIo8AYnHb4McADwLVBthsFjAJo3759omGk1Pode0Mm9meuOYlTOzenRr71MjXGpE9CyV1EauAk9pdU9Q0AVS32LH8C+F+wbVV1HDAOoKioKPzYtxli9rfbeGLqSt5fUDWxX1rUjktOaseJHZqkITJjjKkokd4yAjwFLFLVv3nKW7vt8QAXAOGHQMwCJ//5Q9bv2Bd02e+/152r+nfAOR3GGJMZEqm5nwJcCcwXkXlu2V3A5SLSC6dZZhXwo4QiTKN5a7bz81fnBU3sA7o0Z3D3VlzZz8aDMcZknkR6y0wDglVXs7pP+5KNJWzZtZ8ebRtx/iOfBl3n54OO5hab8s4Yk8HsDlVXWZnyoxdnM3Ghc8kg2L1Gi8cMJU+EGvnWBGOMyWzVPrnv3HeQ/5u8gpYNa5UndoAyzyXeT0efhapSu0Z+GiI0xpjYVfvkft/7S3hhxrchl99zXjfaNK6TwoiMMSZx1b4z9t6DoUdwvLB3G67qX5i6YIwxJkmqdc39H5OW8frstVXKh3ZvxYkdmnDNKYU20JcxJitVq+T+7ZbddGhWD4CF63fy4KSlQdd77MoTUxmWMcYkXbVJ7pMWFnP987O48fROPDZlRcj1XvtR/xRGZYwx/qg2yX3+uh0AYRP7zLsG0rJh7VSFZIwxvqk2F1SjGbzGErsxJldkdc19w469PDBhKUO6t2Lb7gNs3LmP+rUKaFKvBrUK8llaXMLRLRuwaMNO/vnR8qD7qJmfx4FDZfz0rM4pjt4YY/yT1cm9ZF8pr89eG7THSzjn9mzND/t1oFXD2hQ2r8ec1ds4vm1jn6I0xpjUy+rkfnTLBtx5zjHMWb2N+Wt3sH7HPnq2bcSWXQdo06QOLRvW5r9frqdh7QL+fGFPendoTMsGtat0b+zd3obpNcbkFlFN/1DqRUVFOmvWrHSHYYwxWUVEZqtqUbBl1eaCqjHGVCeW3I0xJgdZcjfGmBzkW3IXkaEiskRElovIaL+OY4wxpipfkruI5AOPAOcA3XCm3uvmx7GMMcZU5VfNvQ+wXFVXquoB4BVghE/HMsYYU4lfyb0NsMbzfK1bZowxJgXSdkFVREaJyCwRmbV58+Z0hWGMMTnJrztU1wHtPM/bumXlVHUcMA5ARDaLSOi57iJrDnyXwPZ+sbhiY3HFxuKKTS7G1SHUAl/uUBWRAmApMBAnqX8B/EBVFyT9YM7xZoW6SyudLK7YWFyxsbhiU93i8qXmrqqlInIz8AGQDzztV2I3xhhTlW8Dh6nqu8C7fu3fGGNMaLlyh+q4dAcQgsUVG4srNhZXbKpVXBkxKqQxxpjkypWauzHGGA9L7sYYk4OyOrmnc3AyEWknIpNFZKGILBCRW9zye0RknYjMc3+Geba50411iYgM8TG2VSIy3z3+LLesqYhMFJFl7u8mbrmIyENuXF+JSG+fYurqOSfzRGSniNyajvMlIk+LyCYR+dpTFvP5EZGR7vrLRGSkT3HdJyKL3WO/KSKN3fJCEdnrOW+PebY50f3/L3djl2DHSzCumP9vyX6/hojrVU9Mq0RknlueyvMVKjek9jWmqln5g9PFcgXQEagJfAl0S+HxWwO93ccNcPr1dwPuAX4VZP1uboy1gKPc2PN9im0V0LxS2V+B0e7j0cBf3MfDgPcAAfoBM1P0v9uIcwNGys8XcBrQG/g63vMDNAVWur+buI+b+BDXYKDAffwXT1yF3vUq7edzN1ZxYz/Hh7hi+r/58X4NFlel5Q8Av03D+QqVG1L6GsvmmntaBydT1Q2qOsd9XAIsIvz4OSOAV1R1v6p+AyzH+RtSZQTwnPv4OeB8T/nz6pgBNBaR1j7HMhBYoarh7kr27Xyp6lRga5DjxXJ+hgATVXWrqm4DJgJDkx2Xqk5Q1VL36Qycu71DcmNrqKoz1MkQz3v+lqTFFUao/1vS36/h4nJr35cA48Ptw6fzFSo3pPQ1ls3JPWMGJxORQuAEYKZbdLP79erpwFcvUhuvAhNEZLaIjHLLWqrqBvfxRqBlGuIKuIyKb7p0ny+I/fyk47xdi1PDCzhKROaKyBQRGeCWtXFjSUVcsfzfUn2+BgDFqrrMU5by81UpN6T0NZbNyT0jiEh94N/Araq6E3gU6AT0AjbgfDVMtVNVtTfOePo3ichp3oVuDSUtfWBFpCbwPeBfblEmnK8K0nl+QhGRu4FS4CW3aAPQXlVPAH4BvCwiDVMYUsb93yq5nIoViJSfryC5oVwqXmPZnNwjDk7mNxGpgfPPe0lV3wBQ1WJVPaSqZcATHG5KSFm8qrrO/b0JeNONoTjQ3OL+3pTquFznAHNUtdiNMe3nyxXr+UlZfCJyNXAucIWbFHCbPba4j2fjtGcf7cbgbbrxJa44/m+pPF8FwIXAq554U3q+guUGUvway+bk/gXQRUSOcmuDlwFvp+rgbpveU8AiVf2bp9zbXn0BELiS/zZwmYjUEpGjgC44F3KSHVc9EWkQeIxzQe5r9/iBq+0jgbc8cV3lXrHvB+zwfHX0Q4UaVbrPl0es5+cDYLCINHGbJAa7ZUklIkOB24HvqeoeT3kLcWY8Q0Q64pyflW5sO0Wkn/savcrztyQzrlj/b6l8vw4CFqtqeXNLKs9XqNxAql9jiVwVTvcPzlXmpTifwnen+Nin4nyt+gqY5/4MA14A5rvlbwOtPdvc7ca6hASvyIeJqyNOT4QvgQWB8wI0Az4ElgGTgKZuueBMibjCjbvIx3NWD9gCNPKUpfx84Xy4bAAO4rRjXhfP+cFpA1/u/lzjU1zLcdpdA6+xx9x1L3L/v/OAOcB5nv0U4STbFcDDuHeiJzmumP9vyX6/BovLLX8WuLHSuqk8X6FyQ0pfYzb8gDHG5KBsbpYxxhgTgiV3Y4zJQZbcjTEmB1lyN8aYHGTJ3RhjcpAld2OMyUGW3I0xJgf9P8mSxWaI/fqWAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Episode reward: 222.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Some checks.."
      ],
      "metadata": {
        "id": "vc9AtyT0ij79"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(np.array(g_store).shape, np.array(Values_store).shape, np.array(G_store).shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "avDq5H9rOzjy",
        "outputId": "21ea916c-d209-4a7f-ee35-b59147f36b39"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(64, 100) (64,) (64,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Conclusion: While batch gradient descent with MonteCarlo RL is possible, it isn't advisable because in RL, the policy is being updated and learnt based on the current Q-value estimation. So it is more rfficient to update the policy as soon as possible rather than wait till the end of a batch before the policy is updated, it will require longer training time to get best policies compared to Q-learning and single episode monte-Carlo methods.\n",
        "\n",
        "Also, Q-learning is preferred to MonteCarlo method because the Q-values and policies get updated at the end of each timestep rather than wait till the end of the entire episode. And In the case of function approximation, where the ground truth (true return) itself has to be learnt and updated through experience, it is better to update the Q-values as soon as possible.\n",
        "\n",
        "Finally, we can observe the reward recieved by the agent per batch increasing with the number of iterations."
      ],
      "metadata": {
        "id": "u1Q8oLl7Mckp"
      }
    }
  ]
}