PG_Wandb_sweep.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "PG_Wandb_sweep.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/MoustHolmes/AMAS_Project/blob/main/PG_Wandb_sweep.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7oQTNVeq63Cm"
      },
      "source": [
        "# Installs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NgdLryIc12tF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "663cc309-e72d-42c6-aeff-78dc9a763e74"
      },
      "source": [
        "!pip3 install box2d-py\n",
        "!pip install wandb"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: box2d-py in /usr/local/lib/python3.7/dist-packages (2.3.8)\n",
            "Requirement already satisfied: wandb in /usr/local/lib/python3.7/dist-packages (0.10.23)\n",
            "Requirement already satisfied: six>=1.13.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (1.15.0)\n",
            "Requirement already satisfied: subprocess32>=3.5.3 in /usr/local/lib/python3.7/dist-packages (from wandb) (3.5.4)\n",
            "Requirement already satisfied: configparser>=3.8.1 in /usr/local/lib/python3.7/dist-packages (from wandb) (5.0.2)\n",
            "Requirement already satisfied: docker-pycreds>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (0.4.0)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.7/dist-packages (from wandb) (3.13)\n",
            "Requirement already satisfied: requests<3,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.23.0)\n",
            "Requirement already satisfied: pathtools in /usr/local/lib/python3.7/dist-packages (from wandb) (0.1.2)\n",
            "Requirement already satisfied: promise<3,>=2.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.3)\n",
            "Requirement already satisfied: GitPython>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (3.1.14)\n",
            "Requirement already satisfied: Click>=7.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (7.1.2)\n",
            "Requirement already satisfied: protobuf>=3.12.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (3.12.4)\n",
            "Requirement already satisfied: sentry-sdk>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (1.0.0)\n",
            "Requirement already satisfied: shortuuid>=0.5.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (1.0.1)\n",
            "Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (5.4.8)\n",
            "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.8.1)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (2020.12.5)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (1.24.3)\n",
            "Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.7/dist-packages (from GitPython>=1.0.0->wandb) (4.0.6)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from protobuf>=3.12.0->wandb) (54.1.2)\n",
            "Requirement already satisfied: smmap<4,>=3.0.1 in /usr/local/lib/python3.7/dist-packages (from gitdb<5,>=4.0.1->GitPython>=1.0.0->wandb) (3.0.5)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IL8C74KZ9GqA",
        "outputId": "f225fba6-6345-4247-e9ad-b5bb77f3516f"
      },
      "source": [
        "!wandb login"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mmhp27\u001b[0m (use `wandb login --relogin` to force relogin)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yXeDEL7e7D8o"
      },
      "source": [
        "## Define the Q Learning Agent"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bOpZru0S1fOP",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "afb989c5-c081-402a-ec22-0ae94b84239b"
      },
      "source": [
        "%%writefile Agent.py\n",
        "import torch as T\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "import numpy as np\n",
        "\n",
        "class PolicyNetwork(nn.Module):\n",
        "    def __init__(self, lr, input_dims, n_actions, ):\n",
        "        super(PolicyNetwork, self).__init__()\n",
        "        self.fc1 = nn.Linear(*input_dims, 128)\n",
        "        self.fc2 = nn.Linear(128,128)\n",
        "        self.fc3 = nn.Linear(128, n_actions)\n",
        "        self.optimizer = optim.Adam(self.parameters(), lr=lr)\n",
        "        \n",
        "        self.device = T.device('cuda:0' if T.cuda.is_available() else 'cpu')\n",
        "        self.to(self.device)\n",
        "    def forward(self, state):\n",
        "        x = F.relu(self.fc1(state))\n",
        "        x = F.relu(self.fc2(x))\n",
        "        x = self.fc3(x)\n",
        "        return x\n",
        "\n",
        "class Agent():\n",
        "    def __init__(self, lr, input_dims, n_actions, gamma=0.99):\n",
        "        #self.fc1_dims = fc1_dims\n",
        "        #self.fc2_dims = fc2_dims\n",
        "        self.gamma = gamma\n",
        "        self.lr = lr\n",
        "        self.reward_memory = []\n",
        "        self.action_memory = []\n",
        "        self.policy = PolicyNetwork(self.lr, input_dims, n_actions, fc1_dims=fc1_dims, fc2_dims= fc2_dims)\n",
        "    \n",
        "    def choose_action(self, observation):\n",
        "        state = T.Tensor([observation]).to(self.policy.device)\n",
        "        probabilities = F.softmax(self.policy.forward(state))\n",
        "        action_probs = T.distributions.Categorical(probabilities)\n",
        "        action = action_probs.sample()\n",
        "        log_probs = action_probs.log_prob(action)\n",
        "        self.action_memory.append(log_probs)\n",
        "        return action.item()\n",
        "    \n",
        "    def store_rewards(self, reward):\n",
        "        self.reward_memory.append(reward)\n",
        "    \n",
        "    def learn(self):\n",
        "        self.policy.optimizer.zero_grad()\n",
        "        \n",
        "        # Reward to go:\n",
        "        # G_t = R_t+1 + gamma * R_t+2 + gamma**2 * R_t+3\n",
        "        # G_t = sum from k=0 to k=T {gamma**k * R_t+k+1}\n",
        "        G = np.zeros_like(self.reward_memory, dtype=np.float64)\n",
        "        for t in range(len(self.reward_memory)):\n",
        "            G_sum = 0\n",
        "            discount = 1 #discount factor\n",
        "            for k in range(t, len(self.reward_memory)):\n",
        "                G_sum += self.reward_memory[k] * discount\n",
        "                discount *= self.gamma\n",
        "            G[t] = G_sum\n",
        "        G = T.tensor(G, dtype=T.float).to(self.policy.device)\n",
        "        \n",
        "        loss = 0\n",
        "        for g, logprob in zip(G, self.action_memory):\n",
        "            loss += -g * logprob\n",
        "        loss.backward()\n",
        "        self.policy.optimizer.step()\n",
        "        self.action_memory = []\n",
        "        self.reward_memory = []\n",
        "\n",
        "    def print_args(self):\n",
        "        print('lr       : ' +str(self.lr))\n",
        "        print('gamma    : ' +str(self.gamma))\n",
        "        #print('fc1_dims : ' +str(self.fc1_dims))\n",
        "        #print('fc2_dims : ' +str(self.fc2_dims))"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Overwriting Agent.py\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yCxRAai17JWa"
      },
      "source": [
        "## Train and Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vdzejSm81kwI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "88a4be67-e71d-42e6-8344-a8717c1eb732"
      },
      "source": [
        "%%writefile main.py\n",
        "\n",
        "import numpy as np\n",
        "import wandb\n",
        "from Agent import Agent\n",
        "import argparse\n",
        "import gym \n",
        "import pprint \n",
        "\n",
        "def main():\n",
        "    wandb.init(project='AMAS_Project_Policy_Gradient', config=args)\n",
        "    pprint.pprint(args)\n",
        "    env = gym.make('LunarLander-v2')\n",
        "    agent = Agent(gamma=args.gamma, lr=args.lr,\n",
        "                  input_dims = [8], n_actions = 4)\n",
        "    agent.print_args()\n",
        "    \n",
        "    best_avg_score, weighted_best_score = train(env, agent, episodes = args.episodes, avg_len = args.avg_len, burn_in_time = args.burn_in_time)\n",
        "\n",
        "    wandb.log({'best_avg_score': best_avg_score,'weighted_best_score':weighted_best_score})\n",
        "\n",
        "def train(env, agent, episodes=500, avg_len = 50, burn_in_time = 50):\n",
        "    \"\"\"The play function runs iterations and updates Q-values if desired.\"\"\"\n",
        "\n",
        "    \n",
        "    scores, eps_history = [], []\n",
        "    best_score = float('-inf')\n",
        "\n",
        "    for i in range(episodes):\n",
        "        score = 0\n",
        "        done = False\n",
        "        observation = env.reset()\n",
        "        while not done:\n",
        "            action = agent.choose_action(observation)\n",
        "            observation_, reward, done, info = env.step(action)\n",
        "            score += reward\n",
        "            agent.store_rewards(reward)\n",
        "            agent.learn()\n",
        "            observation = observation_\n",
        "        scores.append(score)\n",
        "        #ep_history.append(.epsilon)\n",
        "\n",
        "        avg_score = np.mean(scores[-avg_len:])\n",
        "        if avg_score > best_score and i > burn_in_time:\n",
        "          best_score = avg_score\n",
        "\n",
        "        print('episode ', i, 'score %.1f' % score,\n",
        "            'average score %.1f' % avg_score, 'best score:%.1f' %best_score)\n",
        "  \n",
        "        wandb.log({'Scores': score,'Avg_Score': avg_score, 'episodes': episodes})\n",
        "\n",
        "    weighted_best_score =np.max( np.array(scores[burn_in_time:])/np.arange(len(scores))[burn_in_time:]) \n",
        "    print(weighted_best_score)\n",
        "    return best_score, weighted_best_score\n",
        "\n",
        "def argumentParser():\n",
        "    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)\n",
        "\n",
        "    parser.add_argument('--gamma',        default = 0.99, type=float, help='Discounting Factor')\n",
        "    parser.add_argument('--lr',          default = 0.0005, type=float, help='Learning Rate')\n",
        "    parser.add_argument('--episodes',     default=2500, type=int, help='number of episodes')\n",
        "    parser.add_argument('--burn_in_time', default=50, type=int, help='number of episodes before calculating avg score')\n",
        "    parser.add_argument('--avg_len',      default=50, type=int, help='number of episodes  is calculated over')\n",
        "    #parser.add_argument('--fc1_dims',     default=256, type=int, help='size of first fully conected layer in the netork')\n",
        "    #parser.add_argument('--fc2_dims',     default=256, type=int, help='ize of second fully conected layer in the netork')\n",
        "    return parser\n",
        "\n",
        "if __name__ == '__main__':\n",
        "  global args\n",
        "  args = argumentParser().parse_args()\n",
        "  main()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Overwriting main.py\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yyMXv-15iApg"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ebg0D9oyiuy1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9da2e1ff-8cea-4caa-86a8-2f92a4ff73b2"
      },
      "source": [
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ArgumentParser(prog='ipykernel_launcher.py', usage=None, description=None, formatter_class=<class 'argparse.ArgumentDefaultsHelpFormatter'>, conflict_handler='error', add_help=True)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HwL4A5lD1tTS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "10f82ee0-575a-40d5-c1ce-3c18df1ae09b"
      },
      "source": [
        "!python3 main.py"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mmoustholmes\u001b[0m (use `wandb login --relogin` to force relogin)\n",
            "2021-03-25 02:31:42.047954: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.10.23\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mvolcanic-feather-7\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/moustholmes/AMAS_Project_Policy_Gradient\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/moustholmes/AMAS_Project_Policy_Gradient/runs/if56usc8\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in /content/wandb/run-20210325_023141-if56usc8\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run `wandb offline` to turn off syncing.\n",
            "\n",
            "Namespace(avg_len=50, burn_in_time=50, episodes=700, fc1_dims=256, fc2_dims=256, gamma=0.99, lr=0.001)\n",
            "lr       : 0.001\n",
            "gamma    : 0.99\n",
            "fc1_dims : 256\n",
            "fc2_dims : 256\n",
            "/content/Agent.py:35: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
            "  probabilities = F.softmax(self.policy.forward(state))\n",
            "episode  0 score -632.8 average score -632.8 best score:-inf\n",
            "episode  1 score -301.3 average score -467.0 best score:-inf\n",
            "episode  2 score -446.3 average score -460.1 best score:-inf\n",
            "episode  3 score -74.5 average score -363.7 best score:-inf\n",
            "episode  4 score -288.6 average score -348.7 best score:-inf\n",
            "episode  5 score -306.0 average score -341.6 best score:-inf\n",
            "episode  6 score -82.7 average score -304.6 best score:-inf\n",
            "episode  7 score 3.8 average score -266.0 best score:-inf\n",
            "episode  8 score -269.9 average score -266.5 best score:-inf\n",
            "episode  9 score -229.1 average score -262.7 best score:-inf\n",
            "episode  10 score 282.3 average score -213.2 best score:-inf\n",
            "episode  11 score -177.7 average score -210.2 best score:-inf\n",
            "episode  12 score -48.8 average score -197.8 best score:-inf\n",
            "episode  13 score -409.2 average score -212.9 best score:-inf\n",
            "episode  14 score -200.0 average score -212.1 best score:-inf\n",
            "episode  15 score -171.0 average score -209.5 best score:-inf\n",
            "episode  16 score -236.9 average score -211.1 best score:-inf\n",
            "episode  17 score -228.1 average score -212.0 best score:-inf\n",
            "episode  18 score -118.0 average score -207.1 best score:-inf\n",
            "episode  19 score 15.3 average score -196.0 best score:-inf\n",
            "episode  20 score -239.8 average score -198.1 best score:-inf\n",
            "episode  21 score 19.3 average score -188.2 best score:-inf\n",
            "episode  22 score -63.9 average score -182.8 best score:-inf\n",
            "episode  23 score 228.7 average score -165.6 best score:-inf\n",
            "episode  24 score -55.6 average score -161.2 best score:-inf\n",
            "episode  25 score -209.6 average score -163.1 best score:-inf\n",
            "episode  26 score -63.4 average score -159.4 best score:-inf\n",
            "episode  27 score -265.5 average score -163.2 best score:-inf\n",
            "episode  28 score -91.6 average score -160.7 best score:-inf\n",
            "episode  29 score -398.8 average score -168.7 best score:-inf\n",
            "episode  30 score -156.2 average score -168.3 best score:-inf\n",
            "episode  31 score -210.3 average score -169.6 best score:-inf\n",
            "episode  32 score -131.4 average score -168.4 best score:-inf\n",
            "episode  33 score -136.4 average score -167.5 best score:-inf\n",
            "episode  34 score 207.3 average score -156.8 best score:-inf\n",
            "episode  35 score -50.9 average score -153.8 best score:-inf\n",
            "episode  36 score -206.8 average score -155.3 best score:-inf\n",
            "episode  37 score -60.2 average score -152.8 best score:-inf\n",
            "episode  38 score -208.6 average score -154.2 best score:-inf\n",
            "episode  39 score 190.4 average score -145.6 best score:-inf\n",
            "episode  40 score -78.0 average score -143.9 best score:-inf\n",
            "episode  41 score -192.7 average score -145.1 best score:-inf\n",
            "episode  42 score -239.3 average score -147.3 best score:-inf\n",
            "episode  43 score -231.5 average score -149.2 best score:-inf\n",
            "episode  44 score 13.4 average score -145.6 best score:-inf\n",
            "episode  45 score -178.9 average score -146.3 best score:-inf\n",
            "episode  46 score 179.9 average score -139.4 best score:-inf\n",
            "episode  47 score -191.1 average score -140.4 best score:-inf\n",
            "episode  48 score -74.9 average score -139.1 best score:-inf\n",
            "episode  49 score -112.7 average score -138.6 best score:-inf\n",
            "episode  50 score -133.9 average score -128.6 best score:-inf\n",
            "episode  51 score -201.7 average score -126.6 best score:-126.6\n",
            "episode  52 score 205.0 average score -113.6 best score:-113.6\n",
            "episode  53 score -35.5 average score -112.8 best score:-112.8\n",
            "episode  54 score -262.3 average score -112.3 best score:-112.3\n",
            "episode  55 score -183.6 average score -109.8 best score:-109.8\n",
            "episode  56 score -208.6 average score -112.3 best score:-109.8\n",
            "episode  57 score -235.0 average score -117.1 best score:-109.8\n",
            "episode  58 score -181.2 average score -115.3 best score:-109.8\n",
            "episode  59 score -233.1 average score -115.4 best score:-109.8\n",
            "episode  60 score 273.6 average score -115.6 best score:-109.8\n",
            "episode  61 score 228.8 average score -107.5 best score:-107.5\n",
            "episode  62 score 219.2 average score -102.1 best score:-102.1\n",
            "episode  63 score -250.9 average score -98.9 best score:-98.9\n",
            "episode  64 score 234.8 average score -90.2 best score:-90.2\n",
            "episode  65 score -58.0 average score -88.0 best score:-88.0\n",
            "episode  66 score -203.2 average score -87.3 best score:-87.3\n",
            "episode  67 score 192.6 average score -78.9 best score:-78.9\n",
            "episode  68 score 180.5 average score -72.9 best score:-72.9\n",
            "episode  69 score -133.9 average score -75.9 best score:-72.9\n",
            "episode  70 score -62.7 average score -72.4 best score:-72.4\n",
            "episode  71 score -105.2 average score -74.9 best score:-72.4\n",
            "episode  72 score -7.9 average score -73.7 best score:-72.4\n",
            "episode  73 score -174.3 average score -81.8 best score:-72.4\n",
            "episode  74 score -50.1 average score -81.7 best score:-72.4\n",
            "episode  75 score -123.4 average score -80.0 best score:-72.4\n",
            "episode  76 score -202.9 average score -82.8 best score:-72.4\n",
            "episode  77 score -219.5 average score -81.8 best score:-72.4\n",
            "episode  78 score 239.7 average score -75.2 best score:-72.4\n",
            "episode  79 score -189.5 average score -71.0 best score:-71.0\n",
            "episode  80 score 7.1 average score -67.8 best score:-67.8\n",
            "episode  81 score -75.0 average score -65.1 best score:-65.1\n",
            "episode  82 score 229.8 average score -57.8 best score:-57.8\n",
            "episode  83 score 218.1 average score -50.7 best score:-50.7\n",
            "episode  84 score -180.1 average score -58.5 best score:-50.7\n",
            "episode  85 score -44.7 average score -58.4 best score:-50.7\n",
            "episode  86 score -37.4 average score -55.0 best score:-50.7\n",
            "episode  87 score -3.2 average score -53.8 best score:-50.7\n",
            "episode  88 score -258.7 average score -54.8 best score:-50.7\n",
            "episode  89 score -265.0 average score -63.9 best score:-50.7\n",
            "episode  90 score -327.3 average score -68.9 best score:-50.7\n",
            "episode  91 score 220.4 average score -60.7 best score:-50.7\n",
            "episode  92 score -426.5 average score -64.4 best score:-50.7\n",
            "episode  93 score -281.3 average score -65.4 best score:-50.7\n",
            "episode  94 score -24.7 average score -66.2 best score:-50.7\n",
            "episode  95 score -339.5 average score -69.4 best score:-50.7\n",
            "episode  96 score -340.7 average score -79.8 best score:-50.7\n",
            "episode  97 score -280.2 average score -81.6 best score:-50.7\n",
            "episode  98 score -326.2 average score -86.6 best score:-50.7\n",
            "episode  99 score -375.1 average score -91.9 best score:-50.7\n",
            "episode  100 score -319.1 average score -95.6 best score:-50.7\n",
            "episode  101 score -329.3 average score -98.1 best score:-50.7\n",
            "episode  102 score -301.3 average score -108.2 best score:-50.7\n",
            "episode  103 score -351.3 average score -114.5 best score:-50.7\n",
            "episode  104 score -59.5 average score -110.5 best score:-50.7\n",
            "episode  105 score -276.7 average score -112.4 best score:-50.7\n",
            "episode  106 score -336.0 average score -114.9 best score:-50.7\n",
            "episode  107 score -336.2 average score -116.9 best score:-50.7\n",
            "episode  108 score -279.5 average score -118.9 best score:-50.7\n",
            "episode  109 score -351.3 average score -121.3 best score:-50.7\n",
            "episode  110 score 260.6 average score -121.5 best score:-50.7\n",
            "episode  111 score -324.8 average score -132.6 best score:-50.7\n",
            "episode  112 score -379.1 average score -144.6 best score:-50.7\n",
            "episode  113 score -320.7 average score -146.0 best score:-50.7\n",
            "episode  114 score -370.8 average score -158.1 best score:-50.7\n",
            "episode  115 score -322.8 average score -163.4 best score:-50.7\n",
            "episode  116 score -380.8 average score -166.9 best score:-50.7\n",
            "episode  117 score -328.5 average score -177.3 best score:-50.7\n",
            "episode  118 score -376.4 average score -188.5 best score:-50.7\n",
            "episode  119 score -399.6 average score -193.8 best score:-50.7\n",
            "episode  120 score -307.3 average score -198.7 best score:-50.7\n",
            "episode  121 score -394.7 average score -204.5 best score:-50.7\n",
            "episode  122 score -444.8 average score -213.2 best score:-50.7\n",
            "episode  123 score -425.8 average score -218.2 best score:-50.7\n",
            "episode  124 score -400.0 average score -225.2 best score:-50.7\n",
            "episode  125 score 265.1 average score -217.5 best score:-50.7\n",
            "episode  126 score -293.2 average score -219.3 best score:-50.7\n",
            "episode  127 score 181.0 average score -211.3 best score:-50.7\n",
            "episode  128 score 215.6 average score -211.7 best score:-50.7\n",
            "episode  129 score -89.5 average score -209.7 best score:-50.7\n",
            "episode  130 score 269.2 average score -204.5 best score:-50.7\n",
            "episode  131 score -196.2 average score -206.9 best score:-50.7\n",
            "episode  132 score -61.2 average score -212.7 best score:-50.7\n",
            "episode  133 score 255.2 average score -212.0 best score:-50.7\n",
            "episode  134 score -174.9 average score -211.9 best score:-50.7\n",
            "episode  135 score -199.0 average score -215.0 best score:-50.7\n",
            "episode  136 score -188.3 average score -218.0 best score:-50.7\n",
            "episode  137 score -32.0 average score -218.6 best score:-50.7\n",
            "episode  138 score 239.9 average score -208.6 best score:-50.7\n",
            "episode  139 score 17.4 average score -203.0 best score:-50.7\n",
            "episode  140 score -220.3 average score -200.8 best score:-50.7\n",
            "episode  141 score -191.4 average score -209.1 best score:-50.7\n",
            "episode  142 score 214.1 average score -196.2 best score:-50.7\n",
            "episode  143 score -201.2 average score -194.6 best score:-50.7\n",
            "episode  144 score 239.9 average score -189.3 best score:-50.7\n",
            "episode  145 score -198.6 average score -186.5 best score:-50.7\n",
            "episode  146 score -10.6 average score -179.9 best score:-50.7\n",
            "episode  147 score -198.4 average score -178.3 best score:-50.7\n",
            "episode  148 score -184.9 average score -175.5 best score:-50.7\n",
            "episode  149 score -194.7 average score -171.9 best score:-50.7\n",
            "episode  150 score 61.1 average score -164.2 best score:-50.7\n",
            "episode  151 score -183.8 average score -161.3 best score:-50.7\n",
            "episode  152 score 217.8 average score -151.0 best score:-50.7\n",
            "episode  153 score -180.5 average score -147.5 best score:-50.7\n",
            "episode  154 score -175.2 average score -149.9 best score:-50.7\n",
            "episode  155 score 244.9 average score -139.4 best score:-50.7\n",
            "episode  156 score -157.5 average score -135.9 best score:-50.7\n",
            "episode  157 score -33.5 average score -129.8 best score:-50.7\n",
            "episode  158 score 239.4 average score -119.4 best score:-50.7\n",
            "episode  159 score 243.7 average score -107.5 best score:-50.7\n",
            "episode  160 score -5.7 average score -112.8 best score:-50.7\n",
            "episode  161 score -174.6 average score -109.8 best score:-50.7\n",
            "episode  162 score -185.7 average score -106.0 best score:-50.7\n",
            "episode  163 score -192.1 average score -103.4 best score:-50.7\n",
            "episode  164 score -39.6 average score -96.8 best score:-50.7\n",
            "episode  165 score -216.7 average score -94.7 best score:-50.7\n",
            "episode  166 score -9.2 average score -87.2 best score:-50.7\n",
            "episode  167 score -15.2 average score -81.0 best score:-50.7\n",
            "episode  168 score -134.0 average score -76.1 best score:-50.7\n",
            "episode  169 score -40.5 average score -68.9 best score:-50.7\n",
            "episode  170 score -165.8 average score -66.1 best score:-50.7\n",
            "episode  171 score -64.6 average score -59.5 best score:-50.7\n",
            "episode  172 score 219.2 average score -46.2 best score:-46.2\n",
            "episode  173 score -207.3 average score -41.8 best score:-41.8\n",
            "episode  174 score -87.5 average score -35.6 best score:-35.6\n",
            "episode  175 score 184.9 average score -37.2 best score:-35.6\n",
            "episode  176 score 214.5 average score -27.0 best score:-27.0\n",
            "episode  177 score -145.3 average score -33.6 best score:-27.0\n",
            "episode  178 score -195.5 average score -41.8 best score:-27.0\n",
            "episode  179 score 214.1 average score -35.7 best score:-27.0\n",
            "episode  180 score 260.6 average score -35.9 best score:-27.0\n",
            "episode  181 score -211.2 average score -36.2 best score:-27.0\n",
            "episode  182 score -218.0 average score -39.3 best score:-27.0\n",
            "episode  183 score -150.9 average score -47.4 best score:-27.0\n",
            "episode  184 score -24.8 average score -44.4 best score:-27.0\n",
            "episode  185 score 263.0 average score -35.2 best score:-27.0\n",
            "episode  186 score -22.5 average score -31.9 best score:-27.0\n",
            "episode  187 score -1.0 average score -31.3 best score:-27.0\n",
            "episode  188 score -177.6 average score -39.6 best score:-27.0\n",
            "episode  189 score -68.5 average score -41.3 best score:-27.0\n",
            "episode  190 score -228.8 average score -41.5 best score:-27.0\n",
            "episode  191 score -151.8 average score -40.7 best score:-27.0\n",
            "episode  192 score -233.3 average score -49.7 best score:-27.0\n",
            "episode  193 score -93.9 average score -47.5 best score:-27.0\n",
            "episode  194 score -31.0 average score -52.9 best score:-27.0\n",
            "episode  195 score -27.5 average score -49.5 best score:-27.0\n",
            "episode  196 score 184.7 average score -45.6 best score:-27.0\n",
            "episode  197 score -210.5 average score -45.9 best score:-27.0\n",
            "episode  198 score -156.0 average score -45.3 best score:-27.0\n",
            "episode  199 score 3.3 average score -41.3 best score:-27.0\n",
            "episode  200 score -189.8 average score -46.3 best score:-27.0\n",
            "episode  201 score 116.7 average score -40.3 best score:-27.0\n",
            "episode  202 score 184.0 average score -41.0 best score:-27.0\n",
            "episode  203 score -187.8 average score -41.1 best score:-27.0\n",
            "episode  204 score -234.3 average score -42.3 best score:-27.0\n",
            "episode  205 score 81.9 average score -45.6 best score:-27.0\n",
            "episode  206 score 201.9 average score -38.4 best score:-27.0\n",
            "episode  207 score -74.4 average score -39.2 best score:-27.0\n",
            "episode  208 score 219.0 average score -39.6 best score:-27.0\n",
            "episode  209 score 248.7 average score -39.5 best score:-27.0\n",
            "episode  210 score -205.7 average score -43.5 best score:-27.0\n",
            "episode  211 score -191.7 average score -43.9 best score:-27.0\n",
            "episode  212 score -227.3 average score -44.7 best score:-27.0\n",
            "episode  213 score -181.3 average score -44.5 best score:-27.0\n",
            "episode  214 score 199.2 average score -39.7 best score:-27.0\n",
            "episode  215 score 280.9 average score -29.8 best score:-27.0\n",
            "episode  216 score -234.4 average score -34.3 best score:-27.0\n",
            "episode  217 score 199.1 average score -30.0 best score:-27.0\n",
            "episode  218 score 179.8 average score -23.7 best score:-23.7\n",
            "episode  219 score -46.4 average score -23.8 best score:-23.7\n",
            "episode  220 score -139.0 average score -23.3 best score:-23.3\n",
            "episode  221 score -21.7 average score -22.4 best score:-22.4\n",
            "episode  222 score 306.9 average score -20.7 best score:-20.7\n",
            "episode  223 score -154.7 average score -19.6 best score:-19.6\n",
            "episode  224 score 218.0 average score -13.5 best score:-13.5\n",
            "episode  225 score 244.2 average score -12.3 best score:-12.3\n",
            "episode  226 score -176.9 average score -20.2 best score:-12.3\n",
            "episode  227 score -8.3 average score -17.4 best score:-12.3\n",
            "episode  228 score 12.3 average score -13.3 best score:-12.3\n",
            "episode  229 score 219.0 average score -13.2 best score:-12.3\n",
            "episode  230 score -212.9 average score -22.6 best score:-12.3\n",
            "episode  231 score -81.3 average score -20.0 best score:-12.3\n",
            "episode  232 score -177.3 average score -19.2 best score:-12.3\n",
            "episode  233 score -275.0 average score -21.7 best score:-12.3\n",
            "episode  234 score -231.0 average score -25.8 best score:-12.3\n",
            "episode  235 score 241.3 average score -26.3 best score:-12.3\n",
            "episode  236 score 297.7 average score -19.8 best score:-12.3\n",
            "episode  237 score -173.3 average score -23.3 best score:-12.3\n",
            "episode  238 score -184.1 average score -23.4 best score:-12.3\n",
            "episode  239 score 167.5 average score -18.7 best score:-12.3\n",
            "episode  240 score -191.5 average score -18.0 best score:-12.3\n",
            "episode  241 score -44.9 average score -15.8 best score:-12.3\n",
            "episode  242 score -196.6 average score -15.1 best score:-12.3\n",
            "episode  243 score -244.0 average score -18.1 best score:-12.3\n",
            "episode  244 score 206.4 average score -13.3 best score:-12.3\n",
            "episode  245 score -244.2 average score -17.7 best score:-12.3\n",
            "episode  246 score -7.2 average score -21.5 best score:-12.3\n",
            "episode  247 score -214.1 average score -21.6 best score:-12.3\n",
            "episode  248 score -240.7 average score -23.3 best score:-12.3\n",
            "episode  249 score -135.4 average score -26.1 best score:-12.3\n",
            "episode  250 score -175.4 average score -25.8 best score:-12.3\n",
            "episode  251 score 210.2 average score -23.9 best score:-12.3\n",
            "episode  252 score 268.7 average score -22.2 best score:-12.3\n",
            "episode  253 score -198.1 average score -22.4 best score:-12.3\n",
            "episode  254 score -33.2 average score -18.4 best score:-12.3\n",
            "episode  255 score 136.8 average score -17.3 best score:-12.3\n",
            "episode  256 score -157.3 average score -24.5 best score:-12.3\n",
            "episode  257 score -62.0 average score -24.2 best score:-12.3\n",
            "episode  258 score -188.6 average score -32.4 best score:-12.3\n",
            "episode  259 score -164.0 average score -40.6 best score:-12.3\n",
            "episode  260 score 226.5 average score -32.0 best score:-12.3\n",
            "episode  261 score -224.6 average score -32.6 best score:-12.3\n",
            "episode  262 score -207.4 average score -32.2 best score:-12.3\n",
            "episode  263 score 233.7 average score -23.9 best score:-12.3\n",
            "episode  264 score -29.8 average score -28.5 best score:-12.3\n",
            "episode  265 score -186.0 average score -37.9 best score:-12.3\n",
            "episode  266 score 291.3 average score -27.3 best score:-12.3\n",
            "episode  267 score -185.2 average score -35.0 best score:-12.3\n",
            "episode  268 score -197.3 average score -42.6 best score:-12.3\n",
            "episode  269 score -203.4 average score -45.7 best score:-12.3\n",
            "episode  270 score 264.0 average score -37.7 best score:-12.3\n",
            "episode  271 score 26.3 average score -36.7 best score:-12.3\n",
            "episode  272 score -67.5 average score -44.2 best score:-12.3\n",
            "episode  273 score 182.8 average score -37.4 best score:-12.3\n",
            "episode  274 score -155.2 average score -44.9 best score:-12.3\n",
            "episode  275 score -220.4 average score -54.2 best score:-12.3\n",
            "episode  276 score -338.1 average score -57.4 best score:-12.3\n",
            "episode  277 score 187.9 average score -53.5 best score:-12.3\n",
            "episode  278 score -119.3 average score -56.1 best score:-12.3\n",
            "episode  279 score -85.7 average score -62.2 best score:-12.3\n",
            "episode  280 score 193.1 average score -54.1 best score:-12.3\n",
            "episode  281 score -126.1 average score -55.0 best score:-12.3\n",
            "episode  282 score 254.1 average score -46.4 best score:-12.3\n",
            "episode  283 score -166.8 average score -44.2 best score:-12.3\n",
            "episode  284 score -149.0 average score -42.6 best score:-12.3\n",
            "episode  285 score 264.5 average score -42.1 best score:-12.3\n",
            "episode  286 score -215.9 average score -52.4 best score:-12.3\n",
            "episode  287 score -93.2 average score -50.8 best score:-12.3\n",
            "episode  288 score -18.4 average score -47.5 best score:-12.3\n",
            "episode  289 score 268.3 average score -45.4 best score:-12.3\n",
            "episode  290 score -229.1 average score -46.2 best score:-12.3\n",
            "episode  291 score -240.4 average score -50.1 best score:-12.3\n",
            "episode  292 score 19.5 average score -45.8 best score:-12.3\n",
            "episode  293 score 210.7 average score -36.7 best score:-12.3\n",
            "episode  294 score 5.4 average score -40.7 best score:-12.3\n",
            "episode  295 score 27.6 average score -35.3 best score:-12.3\n",
            "episode  296 score 218.6 average score -30.8 best score:-12.3\n",
            "episode  297 score 33.1 average score -25.8 best score:-12.3\n",
            "episode  298 score -162.2 average score -24.2 best score:-12.3\n",
            "episode  299 score -26.1 average score -22.1 best score:-12.3\n",
            "episode  300 score -33.4 average score -19.2 best score:-12.3\n",
            "episode  301 score -32.8 average score -24.1 best score:-12.3\n",
            "episode  302 score -158.5 average score -32.6 best score:-12.3\n",
            "episode  303 score 209.9 average score -24.5 best score:-12.3\n",
            "episode  304 score -227.9 average score -28.4 best score:-12.3\n",
            "episode  305 score -184.5 average score -34.8 best score:-12.3\n",
            "episode  306 score -178.3 average score -35.2 best score:-12.3\n",
            "episode  307 score 243.8 average score -29.1 best score:-12.3\n",
            "episode  308 score 197.1 average score -21.4 best score:-12.3\n",
            "episode  309 score -185.6 average score -21.8 best score:-12.3\n",
            "episode  310 score -211.6 average score -30.6 best score:-12.3\n",
            "episode  311 score -219.0 average score -30.4 best score:-12.3\n",
            "episode  312 score -25.8 average score -26.8 best score:-12.3\n",
            "episode  313 score 184.2 average score -27.8 best score:-12.3\n",
            "episode  314 score -210.4 average score -31.4 best score:-12.3\n",
            "episode  315 score 130.3 average score -25.1 best score:-12.3\n",
            "episode  316 score -204.9 average score -35.0 best score:-12.3\n",
            "episode  317 score 222.6 average score -26.9 best score:-12.3\n",
            "episode  318 score 9.4 average score -22.7 best score:-12.3\n",
            "episode  319 score -31.3 average score -19.3 best score:-12.3\n",
            "episode  320 score -160.0 average score -27.8 best score:-12.3\n",
            "episode  321 score -223.9 average score -32.8 best score:-12.3\n",
            "episode  322 score -199.5 average score -35.4 best score:-12.3\n",
            "episode  323 score 198.3 average score -35.1 best score:-12.3\n",
            "episode  324 score -218.7 average score -36.4 best score:-12.3\n",
            "episode  325 score -156.3 average score -35.1 best score:-12.3\n",
            "episode  326 score -59.3 average score -29.5 best score:-12.3\n",
            "episode  327 score 230.4 average score -28.7 best score:-12.3\n",
            "episode  328 score 181.3 average score -22.6 best score:-12.3\n",
            "episode  329 score -32.2 average score -21.6 best score:-12.3\n",
            "episode  330 score -333.1 average score -32.1 best score:-12.3\n",
            "episode  331 score 135.8 average score -26.9 best score:-12.3\n",
            "episode  332 score -78.2 average score -33.5 best score:-12.3\n",
            "episode  333 score -24.5 average score -30.7 best score:-12.3\n",
            "episode  334 score 292.2 average score -21.8 best score:-12.3\n",
            "episode  335 score 205.1 average score -23.0 best score:-12.3\n",
            "episode  336 score -208.2 average score -22.9 best score:-12.3\n",
            "episode  337 score 221.9 average score -16.6 best score:-12.3\n",
            "episode  338 score -113.2 average score -18.5 best score:-12.3\n",
            "episode  339 score -209.5 average score -28.0 best score:-12.3\n",
            "episode  340 score -52.4 average score -24.5 best score:-12.3\n",
            "episode  341 score 195.6 average score -15.8 best score:-12.3\n",
            "episode  342 score 265.4 average score -10.9 best score:-10.9\n",
            "episode  343 score 143.4 average score -12.2 best score:-10.9\n",
            "episode  344 score 231.4 average score -7.7 best score:-7.7\n",
            "episode  345 score 266.7 average score -2.9 best score:-2.9\n",
            "episode  346 score -225.5 average score -11.8 best score:-2.9\n",
            "episode  347 score -69.3 average score -13.8 best score:-2.9\n",
            "episode  348 score 130.8 average score -8.0 best score:-2.9\n",
            "episode  349 score -36.3 average score -8.2 best score:-2.9\n",
            "episode  350 score 240.6 average score -2.7 best score:-2.7\n",
            "episode  351 score -263.8 average score -7.3 best score:-2.7\n",
            "episode  352 score -113.7 average score -6.4 best score:-2.7\n",
            "episode  353 score -37.5 average score -11.4 best score:-2.7\n",
            "episode  354 score -248.0 average score -11.8 best score:-2.7\n",
            "episode  355 score -179.0 average score -11.7 best score:-2.7\n",
            "episode  356 score -24.3 average score -8.6 best score:-2.7\n",
            "episode  357 score -230.6 average score -18.1 best score:-2.7\n",
            "episode  358 score 266.6 average score -16.7 best score:-2.7\n",
            "episode  359 score 202.7 average score -8.9 best score:-2.7\n",
            "episode  360 score -34.7 average score -5.4 best score:-2.7\n",
            "episode  361 score -173.8 average score -4.5 best score:-2.7\n",
            "episode  362 score -19.5 average score -4.3 best score:-2.7\n",
            "episode  363 score -15.8 average score -8.3 best score:-2.7\n",
            "episode  364 score -160.3 average score -7.3 best score:-2.7\n",
            "episode  365 score 1.7 average score -9.9 best score:-2.7\n",
            "episode  366 score -29.0 average score -6.4 best score:-2.7\n",
            "episode  367 score -66.6 average score -12.2 best score:-2.7\n",
            "episode  368 score 244.5 average score -7.5 best score:-2.7\n",
            "episode  369 score -145.0 average score -9.7 best score:-2.7\n",
            "episode  370 score 180.1 average score -2.9 best score:-2.7\n",
            "episode  371 score -151.7 average score -1.5 best score:-1.5\n",
            "episode  372 score -193.6 average score -1.4 best score:-1.4\n",
            "episode  373 score -38.0 average score -6.1 best score:-1.4\n",
            "episode  374 score -40.7 average score -2.5 best score:-1.4\n",
            "episode  375 score -38.8 average score -0.2 best score:-0.2\n",
            "episode  376 score -208.8 average score -3.2 best score:-0.2\n",
            "episode  377 score 221.9 average score -3.4 best score:-0.2\n",
            "episode  378 score -12.4 average score -7.2 best score:-0.2\n",
            "episode  379 score -176.4 average score -10.1 best score:-0.2\n",
            "episode  380 score 194.0 average score 0.4 best score:0.4\n",
            "episode  381 score 162.9 average score 1.0 best score:1.0\n",
            "episode  382 score 226.6 average score 7.1 best score:7.1\n",
            "episode  383 score -38.3 average score 6.8 best score:7.1\n",
            "episode  384 score -40.3 average score 0.1 best score:7.1\n",
            "episode  385 score 197.9 average score -0.0 best score:7.1\n",
            "episode  386 score 275.2 average score 9.7 best score:9.7\n",
            "episode  387 score -179.1 average score 1.6 best score:9.7\n",
            "episode  388 score 302.9 average score 10.0 best score:10.0\n",
            "episode  389 score -71.6 average score 12.7 best score:12.7\n",
            "episode  390 score -36.1 average score 13.0 best score:13.0\n",
            "episode  391 score -171.6 average score 5.7 best score:13.0\n",
            "episode  392 score -11.3 average score 0.2 best score:13.0\n",
            "episode  393 score 2.2 average score -2.7 best score:13.0\n",
            "episode  394 score 253.1 average score -2.2 best score:13.0\n",
            "episode  395 score -68.6 average score -8.9 best score:13.0\n",
            "episode  396 score 287.5 average score 1.3 best score:13.0\n",
            "episode  397 score -162.6 average score -0.5 best score:13.0\n",
            "episode  398 score -41.7 average score -4.0 best score:13.0\n",
            "episode  399 score 254.9 average score 1.8 best score:13.0\n",
            "episode  400 score 283.5 average score 2.7 best score:13.0\n",
            "episode  401 score 242.3 average score 12.8 best score:13.0\n",
            "episode  402 score -213.0 average score 10.8 best score:13.0\n",
            "episode  403 score -184.0 average score 7.9 best score:13.0\n",
            "episode  404 score -29.0 average score 12.3 best score:13.0\n",
            "episode  405 score -183.5 average score 12.2 best score:13.0\n",
            "episode  406 score 44.8 average score 13.6 best score:13.6\n",
            "episode  407 score 2.6 average score 18.2 best score:18.2\n",
            "episode  408 score -204.2 average score 8.8 best score:18.2\n",
            "episode  409 score -166.4 average score 1.4 best score:18.2\n",
            "episode  410 score 193.1 average score 6.0 best score:18.2\n",
            "episode  411 score -29.2 average score 8.9 best score:18.2\n",
            "episode  412 score -191.7 average score 5.4 best score:18.2\n",
            "episode  413 score 208.8 average score 9.9 best score:18.2\n",
            "episode  414 score -180.8 average score 9.5 best score:18.2\n",
            "episode  415 score -233.6 average score 4.8 best score:18.2\n",
            "episode  416 score -206.8 average score 1.3 best score:18.2\n",
            "episode  417 score -200.7 average score -1.4 best score:18.2\n",
            "episode  418 score -404.7 average score -14.4 best score:18.2\n",
            "episode  419 score -391.8 average score -19.3 best score:18.2\n",
            "episode  420 score -304.4 average score -29.0 best score:18.2\n",
            "episode  421 score -347.1 average score -32.9 best score:18.2\n",
            "episode  422 score -241.3 average score -33.9 best score:18.2\n",
            "episode  423 score -311.3 average score -39.4 best score:18.2\n",
            "episode  424 score -333.6 average score -45.2 best score:18.2\n",
            "episode  425 score -197.2 average score -48.4 best score:18.2\n",
            "episode  426 score -318.1 average score -50.6 best score:18.2\n",
            "episode  427 score -273.5 average score -60.5 best score:18.2\n",
            "episode  428 score -304.0 average score -66.3 best score:18.2\n",
            "episode  429 score -312.3 average score -69.0 best score:18.2\n",
            "episode  430 score -240.5 average score -77.7 best score:18.2\n",
            "episode  431 score -359.4 average score -88.2 best score:18.2\n",
            "episode  432 score -440.3 average score -101.5 best score:18.2\n",
            "episode  433 score -386.6 average score -108.5 best score:18.2\n",
            "episode  434 score -330.0 average score -114.3 best score:18.2\n",
            "episode  435 score 77.3 average score -116.7 best score:18.2\n",
            "episode  436 score -341.7 average score -129.0 best score:18.2\n",
            "episode  437 score -238.8 average score -130.2 best score:18.2\n",
            "episode  438 score -23.0 average score -136.7 best score:18.2\n",
            "episode  439 score -272.9 average score -140.7 best score:18.2\n",
            "episode  440 score -232.5 average score -144.7 best score:18.2\n",
            "episode  441 score -211.3 average score -145.5 best score:18.2\n",
            "episode  442 score 228.6 average score -140.7 best score:18.2\n",
            "episode  443 score -205.7 average score -144.8 best score:18.2\n",
            "episode  444 score -40.5 average score -150.7 best score:18.2\n",
            "episode  445 score -51.5 average score -150.4 best score:18.2\n",
            "episode  446 score 224.3 average score -151.6 best score:18.2\n",
            "episode  447 score -188.1 average score -152.1 best score:18.2\n",
            "episode  448 score -209.8 average score -155.5 best score:18.2\n",
            "episode  449 score -30.9 average score -161.2 best score:18.2\n",
            "episode  450 score -35.2 average score -167.6 best score:18.2\n",
            "episode  451 score 277.5 average score -166.9 best score:18.2\n",
            "episode  452 score -15.2 average score -162.9 best score:18.2\n",
            "episode  453 score 209.5 average score -155.1 best score:18.2\n",
            "episode  454 score 41.2 average score -153.6 best score:18.2\n",
            "episode  455 score -205.4 average score -154.1 best score:18.2\n",
            "episode  456 score 227.0 average score -150.4 best score:18.2\n",
            "episode  457 score -141.6 average score -153.3 best score:18.2\n",
            "episode  458 score -182.8 average score -152.9 best score:18.2\n",
            "episode  459 score 44.6 average score -148.7 best score:18.2\n",
            "episode  460 score 19.4 average score -152.2 best score:18.2\n",
            "episode  461 score -30.7 average score -152.2 best score:18.2\n",
            "episode  462 score -174.9 average score -151.8 best score:18.2\n",
            "episode  463 score -180.3 average score -159.6 best score:18.2\n",
            "episode  464 score -58.0 average score -157.2 best score:18.2\n",
            "episode  465 score 244.3 average score -147.6 best score:18.2\n",
            "episode  466 score -213.5 average score -147.7 best score:18.2\n",
            "episode  467 score 179.5 average score -140.1 best score:18.2\n",
            "episode  468 score -89.7 average score -133.8 best score:18.2\n",
            "episode  469 score -237.3 average score -130.8 best score:18.2\n",
            "episode  470 score -226.1 average score -129.2 best score:18.2\n",
            "episode  471 score -53.5 average score -123.3 best score:18.2\n",
            "episode  472 score -68.0 average score -119.9 best score:18.2\n",
            "episode  473 score -205.5 average score -117.7 best score:18.2\n",
            "episode  474 score -283.9 average score -116.7 best score:18.2\n",
            "episode  475 score -51.2 average score -113.8 best score:18.2\n",
            "episode  476 score 297.3 average score -101.5 best score:18.2\n",
            "episode  477 score -7.8 average score -96.2 best score:18.2\n",
            "episode  478 score -55.5 average score -91.2 best score:18.2\n",
            "episode  479 score 254.9 average score -79.9 best score:18.2\n",
            "episode  480 score -219.5 average score -79.5 best score:18.2\n",
            "episode  481 score -110.3 average score -74.5 best score:18.2\n",
            "episode  482 score -223.6 average score -70.2 best score:18.2\n",
            "episode  483 score -192.2 average score -66.3 best score:18.2\n",
            "episode  484 score -155.2 average score -62.8 best score:18.2\n",
            "episode  485 score -179.1 average score -67.9 best score:18.2\n",
            "episode  486 score -203.0 average score -65.1 best score:18.2\n",
            "episode  487 score -200.6 average score -64.4 best score:18.2\n",
            "episode  488 score -179.7 average score -67.5 best score:18.2\n",
            "episode  489 score 58.1 average score -60.9 best score:18.2\n",
            "episode  490 score -215.3 average score -60.5 best score:18.2\n",
            "episode  491 score -173.5 average score -59.8 best score:18.2\n",
            "episode  492 score -174.4 average score -67.8 best score:18.2\n",
            "episode  493 score -154.9 average score -66.8 best score:18.2\n",
            "episode  494 score -34.4 average score -66.7 best score:18.2\n",
            "episode  495 score -263.1 average score -70.9 best score:18.2\n",
            "episode  496 score -198.9 average score -79.4 best score:18.2\n",
            "episode  497 score -290.9 average score -81.4 best score:18.2\n",
            "episode  498 score -87.1 average score -79.0 best score:18.2\n",
            "episode  499 score -170.5 average score -81.8 best score:18.2\n",
            "episode  500 score -172.3 average score -84.5 best score:18.2\n",
            "episode  501 score 192.9 average score -86.2 best score:18.2\n",
            "episode  502 score -228.0 average score -90.5 best score:18.2\n",
            "episode  503 score -220.8 average score -99.1 best score:18.2\n",
            "episode  504 score 223.3 average score -95.4 best score:18.2\n",
            "episode  505 score -224.1 average score -95.8 best score:18.2\n",
            "episode  506 score -199.4 average score -104.3 best score:18.2\n",
            "episode  507 score -167.9 average score -104.9 best score:18.2\n",
            "episode  508 score -208.6 average score -105.4 best score:18.2\n",
            "episode  509 score -52.7 average score -107.3 best score:18.2\n",
            "episode  510 score -199.4 average score -111.7 best score:18.2\n",
            "episode  511 score -144.0 average score -114.0 best score:18.2\n",
            "episode  512 score -218.0 average score -114.8 best score:18.2\n",
            "episode  513 score -544.0 average score -122.1 best score:18.2\n",
            "episode  514 score -246.5 average score -125.9 best score:18.2\n",
            "episode  515 score -218.0 average score -135.1 best score:18.2\n",
            "episode  516 score -318.9 average score -137.2 best score:18.2\n",
            "episode  517 score -3.6 average score -140.9 best score:18.2\n",
            "episode  518 score -47.5 average score -140.0 best score:18.2\n",
            "episode  519 score -61.2 average score -136.5 best score:18.2\n",
            "episode  520 score -399.0 average score -140.0 best score:18.2\n",
            "episode  521 score -366.7 average score -146.2 best score:18.2\n",
            "episode  522 score -329.7 average score -151.5 best score:18.2\n",
            "episode  523 score -336.5 average score -154.1 best score:18.2\n",
            "episode  524 score -72.0 average score -149.9 best score:18.2\n",
            "episode  525 score -244.2 average score -153.7 best score:18.2\n",
            "episode  526 score -262.0 average score -164.9 best score:18.2\n",
            "episode  527 score -253.0 average score -169.8 best score:18.2\n",
            "episode  528 score -90.3 average score -170.5 best score:18.2\n",
            "episode  529 score -76.2 average score -177.1 best score:18.2\n",
            "episode  530 score 230.9 average score -168.1 best score:18.2\n",
            "episode  531 score -16.9 average score -166.2 best score:18.2\n",
            "episode  532 score 233.1 average score -157.1 best score:18.2\n",
            "episode  533 score 215.0 average score -149.0 best score:18.2\n",
            "episode  534 score -183.2 average score -149.5 best score:18.2\n",
            "episode  535 score 299.3 average score -140.0 best score:18.2\n",
            "episode  536 score -186.4 average score -139.6 best score:18.2\n",
            "episode  537 score -106.4 average score -137.7 best score:18.2\n",
            "episode  538 score -184.2 average score -137.8 best score:18.2\n",
            "episode  539 score 122.4 average score -136.6 best score:18.2\n",
            "episode  540 score -91.1 average score -134.1 best score:18.2\n",
            "episode  541 score -199.9 average score -134.6 best score:18.2\n",
            "episode  542 score -216.4 average score -135.4 best score:18.2\n",
            "episode  543 score 260.9 average score -127.1 best score:18.2\n",
            "episode  544 score 275.9 average score -120.9 best score:18.2\n",
            "episode  545 score 217.6 average score -111.3 best score:18.2\n",
            "episode  546 score -236.7 average score -112.1 best score:18.2\n",
            "episode  547 score 286.9 average score -100.5 best score:18.2\n",
            "episode  548 score 14.1 average score -98.5 best score:18.2\n",
            "episode  549 score -43.4 average score -95.9 best score:18.2\n",
            "episode  550 score -202.4 average score -96.5 best score:18.2\n",
            "episode  551 score 24.5 average score -99.9 best score:18.2\n",
            "episode  552 score -224.4 average score -99.8 best score:18.2\n",
            "episode  553 score 283.1 average score -89.8 best score:18.2\n",
            "episode  554 score 184.1 average score -90.5 best score:18.2\n",
            "episode  555 score -187.9 average score -89.8 best score:18.2\n",
            "episode  556 score -201.0 average score -89.8 best score:18.2\n",
            "episode  557 score -1.4 average score -86.5 best score:18.2\n",
            "episode  558 score -153.9 average score -85.4 best score:18.2\n",
            "episode  559 score -129.4 average score -87.0 best score:18.2\n",
            "episode  560 score 275.3 average score -77.5 best score:18.2\n",
            "episode  561 score 225.9 average score -70.1 best score:18.2\n",
            "episode  562 score -259.4 average score -70.9 best score:18.2\n",
            "episode  563 score -39.8 average score -60.8 best score:18.2\n",
            "episode  564 score -33.1 average score -56.5 best score:18.2\n",
            "episode  565 score -165.0 average score -55.5 best score:18.2\n",
            "episode  566 score 264.3 average score -43.8 best score:18.2\n",
            "episode  567 score 69.1 average score -42.4 best score:18.2\n",
            "episode  568 score 233.4 average score -36.7 best score:18.2\n",
            "episode  569 score -168.8 average score -38.9 best score:18.2\n",
            "episode  570 score -181.7 average score -34.6 best score:18.2\n",
            "episode  571 score -32.4 average score -27.9 best score:18.2\n",
            "episode  572 score -177.5 average score -24.8 best score:18.2\n",
            "episode  573 score -165.1 average score -21.4 best score:18.2\n",
            "episode  574 score -223.6 average score -24.4 best score:18.2\n",
            "episode  575 score 19.3 average score -19.2 best score:18.2\n",
            "episode  576 score -32.7 average score -14.6 best score:18.2\n",
            "episode  577 score 258.2 average score -4.3 best score:18.2\n",
            "episode  578 score 206.3 average score 1.6 best score:18.2\n",
            "episode  579 score 68.7 average score 4.5 best score:18.2\n",
            "episode  580 score -177.4 average score -3.7 best score:18.2\n",
            "episode  581 score -219.9 average score -7.7 best score:18.2\n",
            "episode  582 score 231.2 average score -7.8 best score:18.2\n",
            "episode  583 score -175.8 average score -15.6 best score:18.2\n",
            "episode  584 score -184.1 average score -15.6 best score:18.2\n",
            "episode  585 score 213.9 average score -17.3 best score:18.2\n",
            "episode  586 score -144.8 average score -16.5 best score:18.2\n",
            "episode  587 score -2.5 average score -14.4 best score:18.2\n",
            "episode  588 score -33.5 average score -11.4 best score:18.2\n",
            "episode  589 score 173.1 average score -10.4 best score:18.2\n",
            "episode  590 score -34.2 average score -9.2 best score:18.2\n",
            "episode  591 score -120.9 average score -7.7 best score:18.2\n",
            "episode  592 score -37.5 average score -4.1 best score:18.2\n",
            "episode  593 score -57.9 average score -10.5 best score:18.2\n",
            "episode  594 score -235.2 average score -20.7 best score:18.2\n",
            "episode  595 score -62.9 average score -26.3 best score:18.2\n",
            "episode  596 score -179.2 average score -25.1 best score:18.2\n",
            "episode  597 score -174.5 average score -34.4 best score:18.2\n",
            "episode  598 score -234.9 average score -39.3 best score:18.2\n",
            "episode  599 score 190.8 average score -34.7 best score:18.2\n",
            "episode  600 score -140.4 average score -33.4 best score:18.2\n",
            "episode  601 score -196.4 average score -37.8 best score:18.2\n",
            "episode  602 score -56.7 average score -34.5 best score:18.2\n",
            "episode  603 score 219.8 average score -35.8 best score:18.2\n",
            "episode  604 score -174.4 average score -42.9 best score:18.2\n",
            "episode  605 score -147.9 average score -42.1 best score:18.2\n",
            "episode  606 score -201.7 average score -42.1 best score:18.2\n",
            "episode  607 score -217.2 average score -46.5 best score:18.2\n",
            "episode  608 score -56.3 average score -44.5 best score:18.2\n",
            "episode  609 score -235.6 average score -46.6 best score:18.2\n",
            "episode  610 score -40.2 average score -52.9 best score:18.2\n",
            "episode  611 score -90.7 average score -59.3 best score:18.2\n",
            "episode  612 score -292.1 average score -59.9 best score:18.2\n",
            "episode  613 score -213.6 average score -63.4 best score:18.2\n",
            "episode  614 score -52.3 average score -63.8 best score:18.2\n",
            "episode  615 score -281.8 average score -66.1 best score:18.2\n",
            "episode  616 score -270.8 average score -76.8 best score:18.2\n",
            "episode  617 score -219.5 average score -82.6 best score:18.2\n",
            "episode  618 score -234.1 average score -91.9 best score:18.2\n",
            "episode  619 score -224.4 average score -93.1 best score:18.2\n",
            "episode  620 score -182.9 average score -93.1 best score:18.2\n",
            "episode  621 score -178.8 average score -96.0 best score:18.2\n",
            "episode  622 score -163.3 average score -95.7 best score:18.2\n",
            "episode  623 score -235.1 average score -97.1 best score:18.2\n",
            "episode  624 score 219.5 average score -88.3 best score:18.2\n",
            "episode  625 score 214.6 average score -84.4 best score:18.2\n",
            "episode  626 score -142.6 average score -86.6 best score:18.2\n",
            "episode  627 score -16.7 average score -92.1 best score:18.2\n",
            "episode  628 score -232.7 average score -100.8 best score:18.2\n",
            "episode  629 score -44.4 average score -103.1 best score:18.2\n",
            "episode  630 score -8.2 average score -99.7 best score:18.2\n",
            "episode  631 score -195.3 average score -99.2 best score:18.2\n",
            "episode  632 score -35.9 average score -104.6 best score:18.2\n",
            "episode  633 score 249.2 average score -96.1 best score:18.2\n",
            "episode  634 score -49.8 average score -93.4 best score:18.2\n",
            "episode  635 score -6.6 average score -97.8 best score:18.2\n",
            "episode  636 score -185.3 average score -98.6 best score:18.2\n",
            "episode  637 score -247.4 average score -103.5 best score:18.2\n",
            "episode  638 score -54.3 average score -103.9 best score:18.2\n",
            "episode  639 score -245.0 average score -112.3 best score:18.2\n",
            "episode  640 score -17.6 average score -111.9 best score:18.2\n",
            "episode  641 score -222.7 average score -114.0 best score:18.2\n",
            "episode  642 score 93.3 average score -111.4 best score:18.2\n",
            "episode  643 score -22.8 average score -110.7 best score:18.2\n",
            "episode  644 score 5.2 average score -105.9 best score:18.2\n",
            "episode  645 score -159.1 average score -107.8 best score:18.2\n",
            "episode  646 score -226.9 average score -108.7 best score:18.2\n",
            "episode  647 score 205.9 average score -101.1 best score:18.2\n",
            "episode  648 score -214.9 average score -100.7 best score:18.2\n",
            "episode  649 score -249.9 average score -109.5 best score:18.2\n",
            "episode  650 score -34.7 average score -107.4 best score:18.2\n",
            "episode  651 score -59.5 average score -104.7 best score:18.2\n",
            "episode  652 score 0.8 average score -103.5 best score:18.2\n",
            "episode  653 score 232.5 average score -103.3 best score:18.2\n",
            "episode  654 score 266.7 average score -94.5 best score:18.2\n",
            "episode  655 score -134.8 average score -94.2 best score:18.2\n",
            "episode  656 score 258.9 average score -85.0 best score:18.2\n",
            "episode  657 score -65.6 average score -82.0 best score:18.2\n",
            "episode  658 score -191.8 average score -84.7 best score:18.2\n",
            "episode  659 score -41.8 average score -80.8 best score:18.2\n",
            "episode  660 score -144.1 average score -82.9 best score:18.2\n",
            "episode  661 score -75.2 average score -82.6 best score:18.2\n",
            "episode  662 score -157.9 average score -79.9 best score:18.2\n",
            "episode  663 score -178.8 average score -79.2 best score:18.2\n",
            "episode  664 score -32.9 average score -78.8 best score:18.2\n",
            "episode  665 score -195.2 average score -77.1 best score:18.2\n",
            "episode  666 score -13.1 average score -71.9 best score:18.2\n",
            "episode  667 score -208.8 average score -71.7 best score:18.2\n",
            "episode  668 score -62.9 average score -68.3 best score:18.2\n",
            "episode  669 score -38.6 average score -64.5 best score:18.2\n",
            "episode  670 score -279.5 average score -66.5 best score:18.2\n",
            "episode  671 score 197.4 average score -59.0 best score:18.2\n",
            "episode  672 score -234.1 average score -60.4 best score:18.2\n",
            "episode  673 score -256.5 average score -60.8 best score:18.2\n",
            "episode  674 score -316.9 average score -71.5 best score:18.2\n",
            "episode  675 score -241.9 average score -80.7 best score:18.2\n",
            "episode  676 score -180.3 average score -81.4 best score:18.2\n",
            "episode  677 score -113.2 average score -83.3 best score:18.2\n",
            "episode  678 score -249.0 average score -83.7 best score:18.2\n",
            "episode  679 score -292.2 average score -88.6 best score:18.2\n",
            "episode  680 score -375.9 average score -96.0 best score:18.2\n",
            "episode  681 score -383.9 average score -99.7 best score:18.2\n",
            "episode  682 score -117.1 average score -101.4 best score:18.2\n",
            "episode  683 score 73.7 average score -104.9 best score:18.2\n",
            "episode  684 score -201.1 average score -107.9 best score:18.2\n",
            "episode  685 score 237.5 average score -103.0 best score:18.2\n",
            "episode  686 score 21.4 average score -98.9 best score:18.2\n",
            "episode  687 score -202.2 average score -98.0 best score:18.2\n",
            "episode  688 score -206.7 average score -101.0 best score:18.2\n",
            "episode  689 score -46.2 average score -97.1 best score:18.2\n",
            "episode  690 score -190.0 average score -100.5 best score:18.2\n",
            "episode  691 score -184.6 average score -99.7 best score:18.2\n",
            "episode  692 score -206.3 average score -105.7 best score:18.2\n",
            "episode  693 score -38.3 average score -106.0 best score:18.2\n",
            "episode  694 score 221.2 average score -101.7 best score:18.2\n",
            "episode  695 score -185.1 average score -102.2 best score:18.2\n",
            "episode  696 score 265.6 average score -92.4 best score:18.2\n",
            "episode  697 score -159.0 average score -99.7 best score:18.2\n",
            "episode  698 score -0.9 average score -95.4 best score:18.2\n",
            "episode  699 score 4.0 average score -90.3 best score:18.2\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Waiting for W&B process to finish, PID 552\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Program ended successfully.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                                                                \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Find user logs for this run at: /content/wandb/run-20210325_023141-if56usc8/logs/debug.log\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Find internal logs for this run at: /content/wandb/run-20210325_023141-if56usc8/logs/debug-internal.log\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                Scores 3.98677\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:             Avg_Score -90.33345\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:              episodes 700\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:              _runtime 555\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:            _timestamp 1616640056\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                 _step 700\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        best_avg_score 18.24262\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   weighted_best_score 4.55979\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                Scores ▅▇▇▃▄▁▁▇▃▇▃▅█▄▂▅▄▇▄▇▇▄▃▅▁▃▃█▂▃█▅▅▃▇▂▂▃▇▃\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:             Avg_Score ▁▃▄▅▆▆▄▂▃▅▆▇▇▇▇▇▆▇▇▇▇███▇▄▄▆▆▅▄▅▇▇▇▅▅▅▆▅\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:              episodes ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:              _runtime ▁▁▁▂▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇███\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:            _timestamp ▁▁▁▂▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇███\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                 _step ▁▁▁▁▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:        best_avg_score ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   weighted_best_score ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Synced \u001b[33mvolcanic-feather-7\u001b[0m: \u001b[34mhttps://wandb.ai/moustholmes/AMAS_Project_Policy_Gradient/runs/if56usc8\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "plCgWrzN7PAd"
      },
      "source": [
        "## Sweep for Hyperparameter tuning"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_R-TWuGmAYuH",
        "outputId": "6e935a46-6b4f-484b-90eb-1464688f629b"
      },
      "source": [
        "%%writefile sweep.yaml\n",
        "project: \"AMAS_Project_Policy_Gradient\"\n",
        "program: main.py\n",
        "method: bayes\n",
        "metric:\n",
        "  name: best_score\n",
        "  goal: maximize\n",
        "parameters:\n",
        "  lr:\n",
        "    values: [ 0.005, 0.0001, 0.0005, 0.000025, 0.00001]\n",
        "  gamma:\n",
        "    values: [0.999, 0.99, 0.9, 0.5]\n",
        "  fc1_dims:\n",
        "    values: [64, 128, 256, 512]\n",
        "  fc2_dims:\n",
        "    values: [64, 128, 256, 512]"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Writing sweep.yaml\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rX6x77OJXhWB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "206c4c19-4e55-4529-d271-605d519ee1a8"
      },
      "source": [
        "!wandb sweep sweep.yaml"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Creating sweep from: sweep.yaml\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Created sweep with ID: \u001b[33m77sc7tfe\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: View sweep at: \u001b[34m\u001b[4mhttps://wandb.ai/moustholmes/AMAS_Project/sweeps/77sc7tfe\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run sweep agent with: \u001b[33mwandb agent moustholmes/AMAS_Project/77sc7tfe\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NSFbSKvXXluL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e994b138-2d41-4781-8ca7-9a138fc308bd"
      },
      "source": [
        "!wandb agent moustholmes/AMAS_Project_Policy_Gradient/bdmxp80z\n"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Starting wandb agent 🕵️\n",
            "2021-03-26 09:06:21,472 - wandb.wandb_agent - INFO - Running runs: []\n",
            "2021-03-26 09:06:21,658 - wandb.wandb_agent - INFO - Agent received command: run\n",
            "2021-03-26 09:06:21,658 - wandb.wandb_agent - INFO - Agent starting run with config:\n",
            "\tgamma: 0.999\n",
            "\tlr: 0.0001\n",
            "2021-03-26 09:06:21,659 - wandb.wandb_agent - INFO - About to run command: /usr/bin/env python main.py --gamma=0.999 --lr=0.0001\n",
            "Traceback (most recent call last):\n",
            "  File \"main.py\", line 4, in <module>\n",
            "    from Agent import Agent\n",
            "  File \"/content/Agent.py\", line 24\n",
            "    def __init__(self, lr, fc1_dims=128, fc2_dims=128, input_dims, n_actions, gamma=0.99):\n",
            "                ^\n",
            "SyntaxError: non-default argument follows default argument\n",
            "2021-03-26 09:06:26,669 - wandb.wandb_agent - INFO - Running runs: ['j07hjub9']\n",
            "2021-03-26 09:06:26,669 - wandb.wandb_agent - INFO - Cleaning up finished run: j07hjub9\n",
            "2021-03-26 09:06:29,703 - wandb.wandb_agent - INFO - Agent received command: run\n",
            "2021-03-26 09:06:29,703 - wandb.wandb_agent - INFO - Agent starting run with config:\n",
            "\tgamma: 0.999\n",
            "\tlr: 0.005\n",
            "2021-03-26 09:06:29,704 - wandb.wandb_agent - INFO - About to run command: /usr/bin/env python main.py --gamma=0.999 --lr=0.005\n",
            "Traceback (most recent call last):\n",
            "  File \"main.py\", line 4, in <module>\n",
            "    from Agent import Agent\n",
            "  File \"/content/Agent.py\", line 24\n",
            "    def __init__(self, lr, fc1_dims=128, fc2_dims=128, input_dims, n_actions, gamma=0.99):\n",
            "                ^\n",
            "SyntaxError: non-default argument follows default argument\n",
            "2021-03-26 09:06:34,710 - wandb.wandb_agent - INFO - Running runs: ['3ztr1ud4']\n",
            "2021-03-26 09:06:34,710 - wandb.wandb_agent - INFO - Cleaning up finished run: 3ztr1ud4\n",
            "2021-03-26 09:06:34,877 - wandb.wandb_agent - INFO - Agent received command: run\n",
            "2021-03-26 09:06:34,877 - wandb.wandb_agent - INFO - Agent starting run with config:\n",
            "\tgamma: 0.999\n",
            "\tlr: 0.005\n",
            "2021-03-26 09:06:34,879 - wandb.wandb_agent - INFO - About to run command: /usr/bin/env python main.py --gamma=0.999 --lr=0.005\n",
            "Traceback (most recent call last):\n",
            "  File \"main.py\", line 4, in <module>\n",
            "    from Agent import Agent\n",
            "  File \"/content/Agent.py\", line 24\n",
            "    def __init__(self, lr, fc1_dims=128, fc2_dims=128, input_dims, n_actions, gamma=0.99):\n",
            "                ^\n",
            "SyntaxError: non-default argument follows default argument\n",
            "2021-03-26 09:06:39,889 - wandb.wandb_agent - INFO - Running runs: ['5jspqegu']\n",
            "2021-03-26 09:06:39,889 - wandb.wandb_agent - ERROR - Detected 3 failed runs in the first 60 seconds, shutting down.\n",
            "2021-03-26 09:06:39,889 - wandb.wandb_agent - INFO - To disable this check set WANDB_AGENT_DISABLE_FLAPPING=true\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Terminating and syncing runs. Press ctrl-c to kill.\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}