Evaluation.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "2d090f5e",
      "metadata": {
        "scrolled": true,
        "id": "2d090f5e",
        "outputId": "a5dee4f0-a59e-45fa-97ba-1e6bd18c33a1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1.23.2\n",
            "1.12.1+cu102\n"
          ]
        }
      ],
      "source": [
        "import numpy as np\n",
        "print(np.__version__)\n",
        "\n",
        "\n",
        "import torch\n",
        "print(torch.__version__)\n",
        "\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "\n",
        "torch.manual_seed(0)\n",
        "\n",
        "from tqdm.notebook import trange\n",
        "\n",
        "import random\n",
        "import math"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "a097e1b5",
      "metadata": {
        "id": "a097e1b5"
      },
      "outputs": [],
      "source": [
        "class TicTacToe:\n",
        "    def __init__(self):\n",
        "        self.row_count = 3\n",
        "        self.column_count = 3\n",
        "        self.action_size = self.row_count * self.column_count\n",
        "\n",
        "    def __repr__(self):\n",
        "        return \"TicTacToe\"\n",
        "\n",
        "    def get_initial_state(self):\n",
        "        return np.zeros((self.row_count, self.column_count))\n",
        "\n",
        "    def get_next_state(self, state, action, player):\n",
        "        row = action // self.column_count\n",
        "        column = action % self.column_count\n",
        "        state[row, column] = player\n",
        "        return state\n",
        "\n",
        "    def get_valid_moves(self, state):\n",
        "        return (state.reshape(-1) == 0).astype(np.uint8)\n",
        "\n",
        "    def check_win(self, state, action):\n",
        "        if action == None:\n",
        "            return False\n",
        "\n",
        "        row = action // self.column_count\n",
        "        column = action % self.column_count\n",
        "        player = state[row, column]\n",
        "\n",
        "        return (\n",
        "            np.sum(state[row, :]) == player * self.column_count\n",
        "            or np.sum(state[:, column]) == player * self.row_count\n",
        "            or np.sum(np.diag(state)) == player * self.row_count\n",
        "            or np.sum(np.diag(np.flip(state, axis=0))) == player * self.row_count\n",
        "        )\n",
        "\n",
        "    def get_value_and_terminated(self, state, action):\n",
        "        if self.check_win(state, action):\n",
        "            return 1, True\n",
        "        if np.sum(self.get_valid_moves(state)) == 0:\n",
        "            return 0, True\n",
        "        return 0, False\n",
        "\n",
        "    def get_opponent(self, player):\n",
        "        return -player\n",
        "\n",
        "    def get_opponent_value(self, value):\n",
        "        return -value\n",
        "\n",
        "    def change_perspective(self, state, player):\n",
        "        return state * player\n",
        "\n",
        "    def get_encoded_state(self, state):\n",
        "        encoded_state = np.stack(\n",
        "            (state == -1, state == 0, state == 1)\n",
        "        ).astype(np.float32)\n",
        "\n",
        "        if len(state.shape) == 3:\n",
        "            encoded_state = np.swapaxes(encoded_state, 0, 1)\n",
        "\n",
        "        return encoded_state"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "682c4ca4",
      "metadata": {
        "id": "682c4ca4"
      },
      "outputs": [],
      "source": [
        "class ConnectFour:\n",
        "    def __init__(self):\n",
        "        self.row_count = 6\n",
        "        self.column_count = 7\n",
        "        self.action_size = self.column_count\n",
        "        self.in_a_row = 4\n",
        "\n",
        "    def __repr__(self):\n",
        "        return \"ConnectFour\"\n",
        "\n",
        "    def get_initial_state(self):\n",
        "        return np.zeros((self.row_count, self.column_count))\n",
        "\n",
        "    def get_next_state(self, state, action, player):\n",
        "        row = np.max(np.where(state[:, action] == 0))\n",
        "        state[row, action] = player\n",
        "        return state\n",
        "\n",
        "    def get_valid_moves(self, state):\n",
        "        return (state[0] == 0).astype(np.uint8)\n",
        "\n",
        "    def check_win(self, state, action):\n",
        "        if action == None:\n",
        "            return False\n",
        "\n",
        "        row = np.min(np.where(state[:, action] != 0))\n",
        "        column = action\n",
        "        player = state[row][column]\n",
        "\n",
        "        def count(offset_row, offset_column):\n",
        "            for i in range(1, self.in_a_row):\n",
        "                r = row + offset_row * i\n",
        "                c = action + offset_column * i\n",
        "                if (\n",
        "                    r < 0\n",
        "                    or r >= self.row_count\n",
        "                    or c < 0\n",
        "                    or c >= self.column_count\n",
        "                    or state[r][c] != player\n",
        "                ):\n",
        "                    return i - 1\n",
        "            return self.in_a_row - 1\n",
        "\n",
        "        return (\n",
        "            count(1, 0) >= self.in_a_row - 1 # vertical\n",
        "            or (count(0, 1) + count(0, -1)) >= self.in_a_row - 1 # horizontal\n",
        "            or (count(1, 1) + count(-1, -1)) >= self.in_a_row - 1 # top left diagonal\n",
        "            or (count(1, -1) + count(-1, 1)) >= self.in_a_row - 1 # top right diagonal\n",
        "        )\n",
        "\n",
        "    def get_value_and_terminated(self, state, action):\n",
        "        if self.check_win(state, action):\n",
        "            return 1, True\n",
        "        if np.sum(self.get_valid_moves(state)) == 0:\n",
        "            return 0, True\n",
        "        return 0, False\n",
        "\n",
        "    def get_opponent(self, player):\n",
        "        return -player\n",
        "\n",
        "    def get_opponent_value(self, value):\n",
        "        return -value\n",
        "\n",
        "    def change_perspective(self, state, player):\n",
        "        return state * player\n",
        "\n",
        "    def get_encoded_state(self, state):\n",
        "        encoded_state = np.stack(\n",
        "            (state == -1, state == 0, state == 1)\n",
        "        ).astype(np.float32)\n",
        "\n",
        "        if len(state.shape) == 3:\n",
        "            encoded_state = np.swapaxes(encoded_state, 0, 1)\n",
        "\n",
        "        return encoded_state"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "02e5b58b",
      "metadata": {
        "id": "02e5b58b"
      },
      "outputs": [],
      "source": [
        "class ResNet(nn.Module):\n",
        "    def __init__(self, game, num_resBlocks, num_hidden, device):\n",
        "        super().__init__()\n",
        "\n",
        "        self.device = device\n",
        "        self.startBlock = nn.Sequential(\n",
        "            nn.Conv2d(3, num_hidden, kernel_size=3, padding=1),\n",
        "            nn.BatchNorm2d(num_hidden),\n",
        "            nn.ReLU()\n",
        "        )\n",
        "\n",
        "        self.backBone = nn.ModuleList(\n",
        "            [ResBlock(num_hidden) for i in range(num_resBlocks)]\n",
        "        )\n",
        "\n",
        "        self.policyHead = nn.Sequential(\n",
        "            nn.Conv2d(num_hidden, 32, kernel_size=3, padding=1),\n",
        "            nn.BatchNorm2d(32),\n",
        "            nn.ReLU(),\n",
        "            nn.Flatten(),\n",
        "            nn.Linear(32 * game.row_count * game.column_count, game.action_size)\n",
        "        )\n",
        "\n",
        "        self.valueHead = nn.Sequential(\n",
        "            nn.Conv2d(num_hidden, 3, kernel_size=3, padding=1),\n",
        "            nn.BatchNorm2d(3),\n",
        "            nn.ReLU(),\n",
        "            nn.Flatten(),\n",
        "            nn.Linear(3 * game.row_count * game.column_count, 1),\n",
        "            nn.Tanh()\n",
        "        )\n",
        "\n",
        "        self.to(device)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.startBlock(x)\n",
        "        for resBlock in self.backBone:\n",
        "            x = resBlock(x)\n",
        "        policy = self.policyHead(x)\n",
        "        value = self.valueHead(x)\n",
        "        return policy, value\n",
        "\n",
        "\n",
        "class ResBlock(nn.Module):\n",
        "    def __init__(self, num_hidden):\n",
        "        super().__init__()\n",
        "        self.conv1 = nn.Conv2d(num_hidden, num_hidden, kernel_size=3, padding=1)\n",
        "        self.bn1 = nn.BatchNorm2d(num_hidden)\n",
        "        self.conv2 = nn.Conv2d(num_hidden, num_hidden, kernel_size=3, padding=1)\n",
        "        self.bn2 = nn.BatchNorm2d(num_hidden)\n",
        "\n",
        "    def forward(self, x):\n",
        "        residual = x\n",
        "        x = F.relu(self.bn1(self.conv1(x)))\n",
        "        x = self.bn2(self.conv2(x))\n",
        "        x += residual\n",
        "        x = F.relu(x)\n",
        "        return x\n",
        ""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "797c3205",
      "metadata": {
        "id": "797c3205",
        "outputId": "0d84d4bf-b064-482d-d230-27e527c59434"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0.9852449893951416\n",
            "[[ 0.  0. -1.]\n",
            " [ 0. -1.  0.]\n",
            " [ 1.  0.  1.]]\n",
            "tensor([[[[0., 0., 1.],\n",
            "          [0., 1., 0.],\n",
            "          [0., 0., 0.]],\n",
            "\n",
            "         [[1., 1., 0.],\n",
            "          [1., 0., 1.],\n",
            "          [0., 1., 0.]],\n",
            "\n",
            "         [[0., 0., 0.],\n",
            "          [0., 0., 0.],\n",
            "          [1., 0., 1.]]]], device='cuda:0')\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAfO0lEQVR4nO3de2zV9f3H8Vdb6CkILZeOU6hH623DirbQ0q4QxWRHu42ZkThXja7NmfYPBYeezNh6aacoB6c2NdBRYXRbVEKn87bh6tjZ0DFriq1s4gXiDLRezmkbtQdrcmrOOb8/9vOwSgscKL57eT6Sb7J++XzPeZ99t/SZ7/me06RYLBYTAACAkWTrAQAAwMRGjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFOTrAc4HtFoVB9++KGmT5+upKQk63EAAMBxiMViOnTokObNm6fk5OGvf4yJGPnwww/lcrmsxwAAACegq6tLp59++rD/PiZiZPr06ZL++2LS09ONpwEAAMcjFArJ5XLFf48PZ0zEyJdvzaSnpxMjAACMMce6xYIbWAEAgCliBAAAmCJGAACAKWIEAACYIkYAAIApYgQAAJgiRgAAgCliBAAAmCJGAACAKWIEAACYIkYAAICpE4qRhoYG5eTkKC0tTcXFxWpraxt27aWXXqqkpKQjtuXLl5/w0AAAYPxIOEaam5vl9XpVW1urjo4O5eXlqbS0VN3d3UOuf/rpp/XRRx/Ft7179yolJUVXXXXVSQ8PAADGvoRjpK6uTpWVlfJ4PMrNzVVjY6OmTp2qpqamIdfPmjVLWVlZ8W3Hjh2aOnUqMQIAACRJkxJZPDAwoPb2dlVXV8f3JScny+12q7W19bgeY8uWLbr66qt12mmnDbsmHA4rHA7Hfw6FQomMCQAY43KqtluPcEwH1nG7wUhJ6MpIb2+vIpGInE7noP1Op1OBQOCYx7e1tWnv3r264YYbjrrO5/MpIyMjvrlcrkTGBAAAY8jX+mmaLVu26MILL1RRUdFR11VXV6uvry++dXV1fU0TAgCAr1tCb9NkZmYqJSVFwWBw0P5gMKisrKyjHtvf369t27bp3nvvPebzOBwOORyOREYDAABjVEJXRlJTU1VQUCC/3x/fF41G5ff7VVJSctRjn3zySYXDYV133XUnNikAABiXEroyIkler1cVFRUqLCxUUVGR6uvr1d/fL4/HI0kqLy9Xdna2fD7foOO2bNmiFStWaPbs2SMzOQAAGBcSjpGysjL19PSopqZGgUBA+fn5amlpid/U2tnZqeTkwRdc9u3bp127dukvf/nLyEwNAADGjaRYLBazHuJYQqGQMjIy1NfXp/T0dOtxAACnGB/tHR+O9/c3f5sGAACYIkYAAIApYgQAAJgiRgAAgCliBAAAmCJGAACAKWIEAACYIkYAAIApYgQAAJgiRgAAgCliBAAAmCJGAACAKWIEAACYIkYAAIApYgQAAJgiRgAAgCliBAAAmCJGAACAKWIEAACYIkYAAIApYgQAAJgiRgAAgCliBAAAmCJGAACAKWIEAACYIkYAAIApYgQAAJgiRgAAgCliBAAAmCJGAACAKWIEAACYIkYAAIApYgQAAJgiRgAAgCliBAAAmCJGAACAKWIEAACYIkYAAIApYgQAAJg6oRhpaGhQTk6O0tLSVFxcrLa2tqOu//TTT7Vy5UrNnTtXDodD3/zmN/XCCy+c0MAAAGB8mZToAc3NzfJ6vWpsbFRxcbHq6+tVWlqqffv2ac6cOUesHxgY0GWXXaY5c+boqaeeUnZ2tg4ePKgZM2aMxPwAAGCMSzhG6urqVFlZKY/HI0lqbGzU9u3b1dTUpKqqqiPWNzU16eOPP9Yrr7yiyZMnS5JycnJObmoAADBuJPQ2zcDAgNrb2+V2uw8/QHKy3G63Wltbhzzm+eefV0lJiVauXCmn06kFCxZo7dq1ikQiwz5POBxWKBQatAEAgPEpoRjp7e1VJBKR0+kctN/pdCoQCAx5zHvvvaennnpKkUhEL7zwgu6++249/PDDuu+++4Z9Hp/Pp4yMjPjmcrkSGRMAAIwhp/zTNNFoVHPmzNGmTZtUUFCgsrIy3XnnnWpsbBz2mOrqavX19cW3rq6uUz0mAAAwktA9I5mZmUpJSVEwGBy0PxgMKisra8hj5s6dq8mTJyslJSW+7/zzz1cgENDAwIBSU1OPOMbhcMjhcCQyGgAAGKMSujKSmpqqgoIC+f3++L5oNCq/36+SkpIhj1m6dKneffddRaPR+L79+/dr7ty5Q4YIAACYWBJ+m8br9Wrz5s363e9+p7fffls33nij+vv745+uKS8vV3V1dXz9jTfeqI8//lirV6/W/v37tX37dq1du1YrV64cuVcBAADGrIQ/2ltWVqaenh7V1NQoEAgoPz9fLS0t8ZtaOzs7lZx8uHFcLpdefPFF3XrrrbrooouUnZ2t1atX6/bbbx+5VwEAAMaspFgsFrMe4lhCoZAyMjLU19en9PR063EAAKdYTtV26xGO6cC65dYjjHrH+/ubv00DAABMESMAAMAUMQIAAEwRIwAAwBQxAgAATBEjAADAFDECAABMESMAAMAUMQIAAEwRIwAAwBQxAgAATBEjAADAFDECAABMESMAAMAUMQIAAEwRIwAAwBQxAgAATBEjAADAFDECAABMESMAAMAUMQIAAEwRIwAAwBQxAgAATBEjAADAFDECAABMESMAAMAUMQIAAEwRIwAAwBQxAgAATBEjAADAFDECAABMESMAAMAUMQIAAEwRIwAAwBQxAgAATBEjAADAFDECAABMESMAAMAUMQIAAEydUIw0NDQoJydHaWlpKi4uVltb27Brf/vb3yopKWnQlpaWdsIDAwCA8SXhGGlubpbX61Vtba06OjqUl5en0tJSdXd3D3tMenq6Pvroo/h28ODBkxoaAACMHwnHSF1dnSorK+XxeJSbm6vGxkZNnTpVTU1Nwx6TlJSkrKys+OZ0Ok9qaAAAMH4kFCMDAwNqb2+X2+0+/ADJyXK73WptbR32uM8++0xnnnmmXC6XfvjDH+rNN9886vOEw2GFQqFBGwAAGJ8SipHe3l5FIpEjrmw4nU4FAoEhj/nWt76lpqYmPffcc3r88ccVjUa1ZMkSvf/++8M+j8/nU0ZGRnxzuVyJjAkAAMaQU/5pmpKSEpWXlys/P1/Lli3T008/rW984xt69NFHhz2murpafX198a2rq+tUjwkAAIxMSmRxZmamUlJSFAwGB+0PBoPKyso6rseYPHmyFi5cqHfffXfYNQ6HQw6HI5HRAADAGJXQlZHU1FQVFBTI7/fH90WjUfn9fpWUlBzXY0QiEb3xxhuaO3duYpMCAIBxKaErI5Lk9XpVUVGhwsJCFRUVqb6+Xv39/fJ4PJKk8vJyZWdny+fzSZLuvfdeffvb39a5556rTz/9VA8++KAOHjyoG264YWRfCQAAGJMSjpGysjL19PSopqZGgUBA+fn5amlpid/U2tnZqeTkwxdcPvnkE1VWVioQCGjmzJkqKCjQK6+8otzc3JF7FQAAYMxKisViMeshjiUUCikjI0N9fX1KT0+3HgcAcIrlVG23HuGYDqxbbj3CqHe8v7/52zQAAMAUMQIAAEwRIwAAwBQxAgAATBEjAADAFDECAABMESMAAMAUMQIAAEwRIwAAwBQxAgAATBEjAADAFDECAABMESMAAMAUMQIAAEwRIwAAwBQxAgAATBEjAADAFDECAABMESMAAMAUMQIAAEwRIwAAwBQxAgAATBEjAADAFDECAABMESMAAMAUMQIAAEwRIwAAwBQxAgAATBEjAADAFDECAABMESMAAMAUMQIAAEwRIwAAwBQxAgAATBEjAADAFDECAABMESMAAMAUMQIAAEwRIwAAwNQJxUhDQ4NycnKUlpam4uJitbW1Hddx27ZtU1JSklasWHEiTwsAAMahhGOkublZXq9XtbW16ujoUF5enkpLS9Xd3X3U4w4cOKCf//znuvjii094WAAAMP4kHCN1dXWqrKyUx+NRbm6uGhsbNXXqVDU1NQ17TCQS0bXXXqt77rlHZ5999kkNDAAAxpeEYmRgYEDt7e1yu92HHyA5WW63W62trcMed++992rOnDm6/vrrT3xSAAAwLk1KZHFvb68ikYicTueg/U6nU++8886Qx+zatUtbtmzRnj17jvt5wuGwwuFw/OdQKJTImAAAYAw5pZ+mOXTokH7yk59o8+bNyszMPO7jfD6fMjIy4pvL5TqFUwIAAEsJXRnJzMxUSkqKgsHgoP3BYFBZWVlHrP/Pf/6jAwcO6Iorrojvi0aj/33iSZO0b98+nXPOOUccV11dLa/XG/85FAoRJAAAjFMJxUhqaqoKCgrk9/vjH8+NRqPy+/1atWrVEevnz5+vN954Y9C+u+66S4cOHdIjjzwybGA4HA45HI5ERgMAAGNUQjEiSV6vVxUVFSosLFRRUZHq6+vV398vj8cjSSovL1d2drZ8Pp/S0tK0YMGCQcfPmDFDko7YDwAAJqaEY6SsrEw9PT2qqalRIBBQfn6+Wlpa4je1dnZ2KjmZL3YFAADHJykWi8WshziWUCikjIwM9fX1KT093XocAMApllO13XqEYzqwbrn1CKPe8f7+5hIGAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFMnFCMNDQ3KyclRWlqaiouL1dbWNuzap59+WoWFhZoxY4ZOO+005efn67HHHjvhgQEAwPiScIw0NzfL6/WqtrZWHR0dysvLU2lpqbq7u4dcP2vWLN15551qbW3Vv//9b3k8Hnk8Hr344osnPTwAABj7kmKxWCyRA4qLi7V48WJt2LBBkhSNRuVyuXTzzTerqqrquB5j0aJFWr58udasWXNc60OhkDIyMtTX16f09PRExgUAjEE5VdutRzimA+uWW48w6h3v7++ErowMDAyovb1dbrf78AMkJ8vtdqu1tfWYx8diMfn9fu3bt0+XXHLJsOvC4bBCodCgDQAAjE8JxUhvb68ikYicTueg/U6nU4FAYNjj+vr6NG3aNKWmpmr58uVav369LrvssmHX+3w+ZWRkxDeXy5XImAAAYAz5Wj5NM336dO3Zs0e7d+/W/fffL6/Xq507dw67vrq6Wn19ffGtq6vr6xgTAAAYmJTI4szMTKWkpCgYDA7aHwwGlZWVNexxycnJOvfccyVJ+fn5evvtt+Xz+XTppZcOud7hcMjhcCQyGgAAGKMSujKSmpqqgoIC+f3++L5oNCq/36+SkpLjfpxoNKpwOJzIUwMAgHEqoSsjkuT1elVRUaHCwkIVFRWpvr5e/f398ng8kqTy8nJlZ2fL5/NJ+u/9H4WFhTrnnHMUDof1wgsv6LHHHtPGjRtH9pUAAIAxKeEYKSsrU09Pj2pqahQIBJSfn6+Wlpb4Ta2dnZ1KTj58waW/v1833XST3n//fU2ZMkXz58/X448/rrKyspF7FQAAYMxK+HtGLPA9IwAwsfA9I+PDKfmeEQAAgJFGjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwNcl6AGs5VdutRzimA+uWW48AAMApw5URAABgihgBAACmiBEAAGCKGAEAAKaIEQAAYIoYAQAApogRAABgihgBAACmiBEAAGCKGAEAAKaIEQAAYIoYAQAApogRAABgihgBAACmiBEAAGCKGAEAAKaIEQAAYIoYAQAApogRAABgihgBAACmiBEAAGDqhGKkoaFBOTk5SktLU3Fxsdra2oZdu3nzZl188cWaOXOmZs6cKbfbfdT1AABgYkk4Rpqbm+X1elVbW6uOjg7l5eWptLRU3d3dQ67fuXOnrrnmGv39739Xa2urXC6XLr/8cn3wwQcnPTwAABj7Eo6Ruro6VVZWyuPxKDc3V42NjZo6daqampqGXP/EE0/opptuUn5+vubPn69f//rXikaj8vv9Jz08AAAY+xKKkYGBAbW3t8vtdh9+gORkud1utba2HtdjfP755/riiy80a9asYdeEw2GFQqFBGwAAGJ8SipHe3l5FIhE5nc5B+51OpwKBwHE9xu2336558+YNCpqv8vl8ysjIiG8ulyuRMQEAwBjytX6aZt26ddq2bZueeeYZpaWlDbuuurpafX198a2rq+trnBIAAHydJiWyODMzUykpKQoGg4P2B4NBZWVlHfXYhx56SOvWrdNf//pXXXTRRUdd63A45HA4EhkNAACMUQldGUlNTVVBQcGgm0+/vBm1pKRk2ON++ctfas2aNWppaVFhYeGJTwsAAMadhK6MSJLX61VFRYUKCwtVVFSk+vp69ff3y+PxSJLKy8uVnZ0tn88nSXrggQdUU1OjrVu3KicnJ35vybRp0zRt2rQRfCkAAGAsSjhGysrK1NPTo5qaGgUCAeXn56ulpSV+U2tnZ6eSkw9fcNm4caMGBgb0ox/9aNDj1NbW6he/+MXJTQ8AAMa8hGNEklatWqVVq1YN+W87d+4c9POBAwdO5CkAAMAEwd+mAQAApogRAABgihgBAACmiBEAAGCKGAEAAKaIEQAAYIoYAQAApogRAABgihgBAACmiBEAAGCKGAEAAKaIEQAAYIoYAQAApogRAABgihgBAACmiBEAAGCKGAEAAKaIEQAAYIoYAQAApogRAABgihgBAACmiBEAAGCKGAEAAKaIEQAAYIoYAQAApogRAABgihgBAACmiBEAAGCKGAEAAKaIEQAAYIoYAQAApogRAABgihgBAACmiBEAAGCKGAEAAKaIEQAAYIoYAQAApogRAABgihgBAACmTihGGhoalJOTo7S0NBUXF6utrW3YtW+++aauvPJK5eTkKCkpSfX19Sc6KwAAGIcSjpHm5mZ5vV7V1taqo6NDeXl5Ki0tVXd395DrP//8c5199tlat26dsrKyTnpgAAAwviQcI3V1daqsrJTH41Fubq4aGxs1depUNTU1Dbl+8eLFevDBB3X11VfL4XCc9MAAAGB8SShGBgYG1N7eLrfbffgBkpPldrvV2to6YkOFw2GFQqFBGwAAGJ8SipHe3l5FIhE5nc5B+51OpwKBwIgN5fP5lJGREd9cLteIPTYAABhdRuWnaaqrq9XX1xffurq6rEcCAACnyKREFmdmZiolJUXBYHDQ/mAwOKI3pzocDu4vAQBggkjoykhqaqoKCgrk9/vj+6LRqPx+v0pKSkZ8OAAAMP4ldGVEkrxeryoqKlRYWKiioiLV19erv79fHo9HklReXq7s7Gz5fD5J/73p9a233or/5w8++EB79uzRtGnTdO65547gSwEAAGNRwjFSVlamnp4e1dTUKBAIKD8/Xy0tLfGbWjs7O5WcfPiCy4cffqiFCxfGf37ooYf00EMPadmyZdq5c+fJvwIAADCmJRwjkrRq1SqtWrVqyH/7amDk5OQoFoudyNMAAIAJYFR+mgYAAEwcxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU8QIAAAwRYwAAABTxAgAADBFjAAAAFPECAAAMEWMAAAAU5OsB8DIyqnabj3CUR1Yt9x6BADAKMOVEQAAYIoYAQAApnibBvgajPa3z6SJ+RYa5wUYHU4oRhoaGvTggw8qEAgoLy9P69evV1FR0bDrn3zySd199906cOCAzjvvPD3wwAP6/ve/f8JDAwCORFxhrEr4bZrm5mZ5vV7V1taqo6NDeXl5Ki0tVXd395DrX3nlFV1zzTW6/vrr9frrr2vFihVasWKF9u7de9LDAwCAsS/hGKmrq1NlZaU8Ho9yc3PV2NioqVOnqqmpacj1jzzyiL773e/qtttu0/nnn681a9Zo0aJF2rBhw0kPDwAAxr6E3qYZGBhQe3u7qqur4/uSk5PldrvV2to65DGtra3yer2D9pWWlurZZ58d9nnC4bDC4XD8576+PklSKBRKZNzjEg1/PuKPOdISed2j/fWcinM4Foz28yJNzHMz3s7LeHo94+m1TGRf/ncUi8WOui6hGOnt7VUkEpHT6Ry03+l06p133hnymEAgMOT6QCAw7PP4fD7dc889R+x3uVyJjDtuZNRbTzByxtNrGW84N6PTeDsv4+n1jKfXcqodOnRIGRkZw/77qPw0TXV19aCrKdFoVB9//LFmz56tpKQkw8mOLRQKyeVyqaurS+np6dbj4P9xXkYvzs3oxHkZvcbSuYnFYjp06JDmzZt31HUJxUhmZqZSUlIUDAYH7Q8Gg8rKyhrymKysrITWS5LD4ZDD4Ri0b8aMGYmMai49PX3U/49kIuK8jF6cm9GJ8zJ6jZVzc7QrIl9K6AbW1NRUFRQUyO/3x/dFo1H5/X6VlJQMeUxJScmg9ZK0Y8eOYdcDAICJJeG3abxeryoqKlRYWKiioiLV19erv79fHo9HklReXq7s7Gz5fD5J0urVq7Vs2TI9/PDDWr58ubZt26bXXntNmzZtGtlXAgAAxqSEY6SsrEw9PT2qqalRIBBQfn6+Wlpa4jepdnZ2Kjn58AWXJUuWaOvWrbrrrrt0xx136LzzztOzzz6rBQsWjNyrGEUcDodqa2uPeJsJtjgvoxfnZnTivIxe4/HcJMWO9XkbAACAU4g/lAcAAEwRIwAAwBQxAgAATBEjAADAFDEyghoaGpSTk6O0tDQVFxerra3NeqQJz+fzafHixZo+fbrmzJmjFStWaN++fdZj4SvWrVunpKQk3XLLLdajQNIHH3yg6667TrNnz9aUKVN04YUX6rXXXrMea0KLRCK6++67ddZZZ2nKlCk655xztGbNmmP+zZexghgZIc3NzfJ6vaqtrVVHR4fy8vJUWlqq7u5u69EmtJdeekkrV67Uq6++qh07duiLL77Q5Zdfrv7+fuvR8P92796tRx99VBdddJH1KJD0ySefaOnSpZo8ebL+/Oc/66233tLDDz+smTNnWo82oT3wwAPauHGjNmzYoLffflsPPPCAfvnLX2r9+vXWo40IPto7QoqLi7V48WJt2LBB0n+/mdblcunmm29WVVWV8XT4Uk9Pj+bMmaOXXnpJl1xyifU4E95nn32mRYsW6Ve/+pXuu+8+5efnq76+3nqsCa2qqkr//Oc/9Y9//MN6FPyPH/zgB3I6ndqyZUt835VXXqkpU6bo8ccfN5xsZHBlZAQMDAyovb1dbrc7vi85OVlut1utra2Gk+Gr+vr6JEmzZs0yngSStHLlSi1fvnzQ/3dg6/nnn1dhYaGuuuoqzZkzRwsXLtTmzZutx5rwlixZIr/fr/3790uS/vWvf2nXrl363ve+ZzzZyBiVf7V3rOnt7VUkEol/C+2XnE6n3nnnHaOp8FXRaFS33HKLli5dOm6/AXgs2bZtmzo6OrR7927rUfA/3nvvPW3cuFFer1d33HGHdu/erZ/97GdKTU1VRUWF9XgTVlVVlUKhkObPn6+UlBRFIhHdf//9uvbaa61HGxHECCaMlStXau/evdq1a5f1KBNeV1eXVq9erR07digtLc16HPyPaDSqwsJCrV27VpK0cOFC7d27V42NjcSIod///vd64okntHXrVl1wwQXas2ePbrnlFs2bN29cnBdiZARkZmYqJSVFwWBw0P5gMKisrCyjqfC/Vq1apT/96U96+eWXdfrpp1uPM+G1t7eru7tbixYtiu+LRCJ6+eWXtWHDBoXDYaWkpBhOOHHNnTtXubm5g/adf/75+sMf/mA0ESTptttuU1VVla6++mpJ0oUXXqiDBw/K5/ONixjhnpERkJqaqoKCAvn9/vi+aDQqv9+vkpISw8kQi8W0atUqPfPMM/rb3/6ms846y3okSPrOd76jN954Q3v27IlvhYWFuvbaa7Vnzx5CxNDSpUuP+Pj7/v37deaZZxpNBEn6/PPPB/0RWklKSUlRNBo1mmhkcWVkhHi9XlVUVKiwsFBFRUWqr69Xf3+/PB6P9WgT2sqVK7V161Y999xzmj59ugKBgCQpIyNDU6ZMMZ5u4po+ffoR9+2cdtppmj17NvfzGLv11lu1ZMkSrV27Vj/+8Y/V1tamTZs2adOmTdajTWhXXHGF7r//fp1xxhm64IIL9Prrr6uurk4//elPrUcbGTGMmPXr18fOOOOMWGpqaqyoqCj26quvWo804UkacvvNb35jPRq+YtmyZbHVq1dbj4FYLPbHP/4xtmDBgpjD4YjNnz8/tmnTJuuRJrxQKBRbvXp17IwzzoilpaXFzj777Nidd94ZC4fD1qONCL5nBAAAmOKeEQAAYIoYAQAApogRAABgihgBAACmiBEAAGCKGAEAAKaIEQAAYIoYAQAApogRAABgihgBAACmiBEAAGCKGAEAAKb+D/LMXwWmz49xAAAAAElFTkSuQmCC",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "tictactoe = TicTacToe()\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "state = tictactoe.get_initial_state()\n",
        "state = tictactoe.get_next_state(state, 2, -1)\n",
        "state = tictactoe.get_next_state(state, 4, -1)\n",
        "state = tictactoe.get_next_state(state, 6, 1)\n",
        "state = tictactoe.get_next_state(state, 8, 1)\n",
        "\n",
        "\n",
        "encoded_state = tictactoe.get_encoded_state(state)\n",
        "\n",
        "tensor_state = torch.tensor(encoded_state, device=device).unsqueeze(0)\n",
        "\n",
        "model = ResNet(tictactoe, 4, 64, device=device)\n",
        "model.load_state_dict(torch.load('model_2.pt', map_location=device))\n",
        "model.eval()\n",
        "\n",
        "policy, value = model(tensor_state)\n",
        "value = value.item()\n",
        "policy = torch.softmax(policy, axis=1).squeeze(0).detach().cpu().numpy()\n",
        "\n",
        "print(value)\n",
        "\n",
        "print(state)\n",
        "print(tensor_state)\n",
        "\n",
        "plt.bar(range(tictactoe.action_size), policy)\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "21866526",
      "metadata": {
        "id": "21866526"
      },
      "outputs": [],
      "source": [
        "class Node:\n",
        "    def __init__(self, game, args, state, parent=None, action_taken=None, prior=0, visit_count=0):\n",
        "        self.game = game\n",
        "        self.args = args\n",
        "        self.state = state\n",
        "        self.parent = parent\n",
        "        self.action_taken = action_taken\n",
        "        self.prior = prior\n",
        "\n",
        "        self.children = []\n",
        "\n",
        "        self.visit_count = visit_count\n",
        "        self.value_sum = 0\n",
        "\n",
        "    def is_fully_expanded(self):\n",
        "        return len(self.children) > 0\n",
        "\n",
        "    def select(self):\n",
        "        best_child = None\n",
        "        best_ucb = -np.inf\n",
        "\n",
        "        for child in self.children:\n",
        "            ucb = self.get_ucb(child)\n",
        "            if ucb > best_ucb:\n",
        "                best_child = child\n",
        "                best_ucb = ucb\n",
        "\n",
        "        return best_child\n",
        "\n",
        "    def get_ucb(self, child):\n",
        "        if child.visit_count == 0:\n",
        "            q_value = 0\n",
        "        else:\n",
        "            q_value = 1 - ((child.value_sum / child.visit_count) + 1) / 2\n",
        "        return q_value + self.args['C'] * (math.sqrt(self.visit_count) / (child.visit_count + 1)) * child.prior\n",
        "\n",
        "    def expand(self, policy):\n",
        "        for action, prob in enumerate(policy):\n",
        "            if prob > 0:\n",
        "                child_state = self.state.copy()\n",
        "                child_state = self.game.get_next_state(child_state, action, 1)\n",
        "                child_state = self.game.change_perspective(child_state, player=-1)\n",
        "\n",
        "                child = Node(self.game, self.args, child_state, self, action, prob)\n",
        "                self.children.append(child)\n",
        "\n",
        "        return child\n",
        "\n",
        "    def backpropagate(self, value):\n",
        "        self.value_sum += value\n",
        "        self.visit_count += 1\n",
        "\n",
        "        value = self.game.get_opponent_value(value)\n",
        "        if self.parent is not None:\n",
        "            self.parent.backpropagate(value)\n",
        "\n",
        "\n",
        "class MCTS:\n",
        "    def __init__(self, game, args, model):\n",
        "        self.game = game\n",
        "        self.args = args\n",
        "        self.model = model\n",
        "\n",
        "    @torch.no_grad()\n",
        "    def search(self, state):\n",
        "        root = Node(self.game, self.args, state, visit_count=1)\n",
        "\n",
        "        policy, _ = self.model(\n",
        "            torch.tensor(self.game.get_encoded_state(state), device=self.model.device).unsqueeze(0)\n",
        "        )\n",
        "        policy = torch.softmax(policy, axis=1).squeeze(0).cpu().numpy()\n",
        "        policy = (1 - self.args['dirichlet_epsilon']) * policy + self.args['dirichlet_epsilon'] \\\n",
        "            * np.random.dirichlet([self.args['dirichlet_alpha']] * self.game.action_size)\n",
        "\n",
        "        valid_moves = self.game.get_valid_moves(state)\n",
        "        policy *= valid_moves\n",
        "        policy /= np.sum(policy)\n",
        "        root.expand(policy)\n",
        "\n",
        "        for search in range(self.args['num_searches']):\n",
        "            node = root\n",
        "\n",
        "            while node.is_fully_expanded():\n",
        "                node = node.select()\n",
        "\n",
        "            value, is_terminal = self.game.get_value_and_terminated(node.state, node.action_taken)\n",
        "            value = self.game.get_opponent_value(value)\n",
        "\n",
        "            if not is_terminal:\n",
        "                policy, value = self.model(\n",
        "                    torch.tensor(self.game.get_encoded_state(node.state), device=self.model.device).unsqueeze(0)\n",
        "                )\n",
        "                policy = torch.softmax(policy, axis=1).squeeze(0).cpu().numpy()\n",
        "                valid_moves = self.game.get_valid_moves(node.state)\n",
        "                policy *= valid_moves\n",
        "                policy /= np.sum(policy)\n",
        "\n",
        "                value = value.item()\n",
        "\n",
        "                node.expand(policy)\n",
        "\n",
        "            node.backpropagate(value)\n",
        "\n",
        "\n",
        "        action_probs = np.zeros(self.game.action_size)\n",
        "        for child in root.children:\n",
        "            action_probs[child.action_taken] = child.visit_count\n",
        "        action_probs /= np.sum(action_probs)\n",
        "        return action_probs\n",
        ""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "a3b28ab8",
      "metadata": {
        "id": "a3b28ab8"
      },
      "outputs": [],
      "source": [
        "class AlphaZero:\n",
        "    def __init__(self, model, optimizer, game, args):\n",
        "        self.model = model\n",
        "        self.optimizer = optimizer\n",
        "        self.game = game\n",
        "        self.args = args\n",
        "        self.mcts = MCTS(game, args, model)\n",
        "\n",
        "    def selfPlay(self):\n",
        "        memory = []\n",
        "        player = 1\n",
        "        state = self.game.get_initial_state()\n",
        "\n",
        "        while True:\n",
        "            neutral_state = self.game.change_perspective(state, player)\n",
        "            action_probs = self.mcts.search(neutral_state)\n",
        "\n",
        "            memory.append((neutral_state, action_probs, player))\n",
        "\n",
        "            temperature_action_probs = action_probs ** (1 / self.args['temperature']) # Divide temperature_action_probs with its sum in case of an error\n",
        "            action = np.random.choice(self.game.action_size, p=temperature_action_probs)\n",
        "\n",
        "            state = self.game.get_next_state(state, action, player)\n",
        "\n",
        "            value, is_terminal = self.game.get_value_and_terminated(state, action)\n",
        "\n",
        "            if is_terminal:\n",
        "                returnMemory = []\n",
        "                for hist_neutral_state, hist_action_probs, hist_player in memory:\n",
        "                    hist_outcome = value if hist_player == player else self.game.get_opponent_value(value)\n",
        "                    returnMemory.append((\n",
        "                        self.game.get_encoded_state(hist_neutral_state),\n",
        "                        hist_action_probs,\n",
        "                        hist_outcome\n",
        "                    ))\n",
        "                return returnMemory\n",
        "\n",
        "            player = self.game.get_opponent(player)\n",
        "\n",
        "    def train(self, memory):\n",
        "        random.shuffle(memory)\n",
        "        for batchIdx in range(0, len(memory), self.args['batch_size']):\n",
        "            sample = memory[batchIdx:min(len(memory) - 1, batchIdx + self.args['batch_size'])] # Change to memory[batchIdx:batchIdx+self.args['batch_size']] in case of an error\n",
        "            state, policy_targets, value_targets = zip(*sample)\n",
        "\n",
        "            state, policy_targets, value_targets = np.array(state), np.array(policy_targets), np.array(value_targets).reshape(-1, 1)\n",
        "\n",
        "            state = torch.tensor(state, dtype=torch.float32, device=self.model.device)\n",
        "            policy_targets = torch.tensor(policy_targets, dtype=torch.float32, device=self.model.device)\n",
        "            value_targets = torch.tensor(value_targets, dtype=torch.float32, device=self.model.device)\n",
        "\n",
        "            out_policy, out_value = self.model(state)\n",
        "\n",
        "            policy_loss = F.cross_entropy(out_policy, policy_targets)\n",
        "            value_loss = F.mse_loss(out_value, value_targets)\n",
        "            loss = policy_loss + value_loss\n",
        "\n",
        "            self.optimizer.zero_grad()\n",
        "            loss.backward()\n",
        "            self.optimizer.step()\n",
        "\n",
        "    def learn(self):\n",
        "        for iteration in range(self.args['num_iterations']):\n",
        "            memory = []\n",
        "\n",
        "            self.model.eval()\n",
        "            for selfPlay_iteration in trange(self.args['num_selfPlay_iterations']):\n",
        "                memory += self.selfPlay()\n",
        "\n",
        "            self.model.train()\n",
        "            for epoch in trange(self.args['num_epochs']):\n",
        "                self.train(memory)\n",
        "\n",
        "            torch.save(self.model.state_dict(), f\"model_{iteration}_{self.game}.pt\")\n",
        "            torch.save(self.optimizer.state_dict(), f\"optimizer_{iteration}_{self.game}.pt\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e997f3ea",
      "metadata": {
        "id": "e997f3ea"
      },
      "outputs": [],
      "source": [
        "class MCTSParallel:\n",
        "    def __init__(self, game, args, model):\n",
        "        self.game = game\n",
        "        self.args = args\n",
        "        self.model = model\n",
        "\n",
        "    @torch.no_grad()\n",
        "    def search(self, states, spGames):\n",
        "        policy, _ = self.model(\n",
        "            torch.tensor(self.game.get_encoded_state(states), device=self.model.device)\n",
        "        )\n",
        "        policy = torch.softmax(policy, axis=1).cpu().numpy()\n",
        "        policy = (1 - self.args['dirichlet_epsilon']) * policy + self.args['dirichlet_epsilon'] \\\n",
        "            * np.random.dirichlet([self.args['dirichlet_alpha']] * self.game.action_size, size=policy.shape[0])\n",
        "\n",
        "        for i, spg in enumerate(spGames):\n",
        "            spg_policy = policy[i]\n",
        "            valid_moves = self.game.get_valid_moves(states[i])\n",
        "            spg_policy *= valid_moves\n",
        "            spg_policy /= np.sum(spg_policy)\n",
        "\n",
        "            spg.root = Node(self.game, self.args, states[i], visit_count=1)\n",
        "            spg.root.expand(spg_policy)\n",
        "\n",
        "        for search in range(self.args['num_searches']):\n",
        "            for spg in spGames:\n",
        "                spg.node = None\n",
        "                node = spg.root\n",
        "\n",
        "                while node.is_fully_expanded():\n",
        "                    node = node.select()\n",
        "\n",
        "                value, is_terminal = self.game.get_value_and_terminated(node.state, node.action_taken)\n",
        "                value = self.game.get_opponent_value(value)\n",
        "\n",
        "                if is_terminal:\n",
        "                    node.backpropagate(value)\n",
        "\n",
        "                else:\n",
        "                    spg.node = node\n",
        "\n",
        "            expandable_spGames = [mappingIdx for mappingIdx in range(len(spGames)) if spGames[mappingIdx].node is not None]\n",
        "\n",
        "            if len(expandable_spGames) > 0:\n",
        "                states = np.stack([spGames[mappingIdx].node.state for mappingIdx in expandable_spGames])\n",
        "\n",
        "                policy, value = self.model(\n",
        "                    torch.tensor(self.game.get_encoded_state(states), device=self.model.device)\n",
        "                )\n",
        "                policy = torch.softmax(policy, axis=1).cpu().numpy()\n",
        "                value = value.cpu().numpy()\n",
        "\n",
        "            for i, mappingIdx in enumerate(expandable_spGames):\n",
        "                node = spGames[mappingIdx].node\n",
        "                spg_policy, spg_value = policy[i], value[i]\n",
        "\n",
        "                valid_moves = self.game.get_valid_moves(node.state)\n",
        "                spg_policy *= valid_moves\n",
        "                spg_policy /= np.sum(spg_policy)\n",
        "\n",
        "                node.expand(spg_policy)\n",
        "                node.backpropagate(spg_value)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "7d0a5a7d",
      "metadata": {
        "id": "7d0a5a7d"
      },
      "outputs": [],
      "source": [
        "class AlphaZeroParallel:\n",
        "    def __init__(self, model, optimizer, game, args):\n",
        "        self.model = model\n",
        "        self.optimizer = optimizer\n",
        "        self.game = game\n",
        "        self.args = args\n",
        "        self.mcts = MCTSParallel(game, args, model)\n",
        "\n",
        "    def selfPlay(self):\n",
        "        return_memory = []\n",
        "        player = 1\n",
        "        spGames = [SPG(self.game) for spg in range(self.args['num_parallel_games'])]\n",
        "\n",
        "        while len(spGames) > 0:\n",
        "            states = np.stack([spg.state for spg in spGames])\n",
        "            neutral_states = self.game.change_perspective(states, player)\n",
        "\n",
        "            self.mcts.search(neutral_states, spGames)\n",
        "\n",
        "            for i in range(len(spGames))[::-1]:\n",
        "                spg = spGames[i]\n",
        "\n",
        "                action_probs = np.zeros(self.game.action_size)\n",
        "                for child in spg.root.children:\n",
        "                    action_probs[child.action_taken] = child.visit_count\n",
        "                action_probs /= np.sum(action_probs)\n",
        "\n",
        "                spg.memory.append((spg.root.state, action_probs, player))\n",
        "\n",
        "                temperature_action_probs = action_probs ** (1 / self.args['temperature'])\n",
        "                action = np.random.choice(self.game.action_size, p=temperature_action_probs) # Divide temperature_action_probs with its sum in case of an error\n",
        "\n",
        "                spg.state = self.game.get_next_state(spg.state, action, player)\n",
        "\n",
        "                value, is_terminal = self.game.get_value_and_terminated(spg.state, action)\n",
        "\n",
        "                if is_terminal:\n",
        "                    for hist_neutral_state, hist_action_probs, hist_player in spg.memory:\n",
        "                        hist_outcome = value if hist_player == player else self.game.get_opponent_value(value)\n",
        "                        return_memory.append((\n",
        "                            self.game.get_encoded_state(hist_neutral_state),\n",
        "                            hist_action_probs,\n",
        "                            hist_outcome\n",
        "                        ))\n",
        "                    del spGames[i]\n",
        "\n",
        "            player = self.game.get_opponent(player)\n",
        "\n",
        "        return return_memory\n",
        "\n",
        "    def train(self, memory):\n",
        "        random.shuffle(memory)\n",
        "        for batchIdx in range(0, len(memory), self.args['batch_size']):\n",
        "            sample = memory[batchIdx:min(len(memory) - 1, batchIdx + self.args['batch_size'])] # Change to memory[batchIdx:batchIdx+self.args['batch_size']] in case of an error\n",
        "            state, policy_targets, value_targets = zip(*sample)\n",
        "\n",
        "            state, policy_targets, value_targets = np.array(state), np.array(policy_targets), np.array(value_targets).reshape(-1, 1)\n",
        "\n",
        "            state = torch.tensor(state, dtype=torch.float32, device=self.model.device)\n",
        "            policy_targets = torch.tensor(policy_targets, dtype=torch.float32, device=self.model.device)\n",
        "            value_targets = torch.tensor(value_targets, dtype=torch.float32, device=self.model.device)\n",
        "\n",
        "            out_policy, out_value = self.model(state)\n",
        "\n",
        "            policy_loss = F.cross_entropy(out_policy, policy_targets)\n",
        "            value_loss = F.mse_loss(out_value, value_targets)\n",
        "            loss = policy_loss + value_loss\n",
        "\n",
        "            self.optimizer.zero_grad()\n",
        "            loss.backward()\n",
        "            self.optimizer.step()\n",
        "\n",
        "    def learn(self):\n",
        "        for iteration in range(self.args['num_iterations']):\n",
        "            memory = []\n",
        "\n",
        "            self.model.eval()\n",
        "            for selfPlay_iteration in trange(self.args['num_selfPlay_iterations'] // self.args['num_parallel_games']):\n",
        "                memory += self.selfPlay()\n",
        "\n",
        "            self.model.train()\n",
        "            for epoch in trange(self.args['num_epochs']):\n",
        "                self.train(memory)\n",
        "\n",
        "            torch.save(self.model.state_dict(), f\"model_{iteration}_{self.game}.pt\")\n",
        "            torch.save(self.optimizer.state_dict(), f\"optimizer_{iteration}_{self.game}.pt\")\n",
        "\n",
        "class SPG:\n",
        "    def __init__(self, game):\n",
        "        self.state = game.get_initial_state()\n",
        "        self.memory = []\n",
        "        self.root = None\n",
        "        self.node = None"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "24bd91ef",
      "metadata": {
        "id": "24bd91ef",
        "outputId": "62ea257c-13eb-40ff-f06b-0e1a669abd6b",
        "colab": {
          "referenced_widgets": [
            "c5668222b9524533b76d7e50618cfaa5"
          ]
        }
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c5668222b9524533b76d7e50618cfaa5",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/5 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "ename": "KeyboardInterrupt",
          "evalue": "",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "Cell \u001b[0;32mIn[10], line 23\u001b[0m\n\u001b[1;32m      9\u001b[0m args \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m     10\u001b[0m     \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mC\u001b[39m\u001b[38;5;124m'\u001b[39m: \u001b[38;5;241m2\u001b[39m,\n\u001b[1;32m     11\u001b[0m     \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mnum_searches\u001b[39m\u001b[38;5;124m'\u001b[39m: \u001b[38;5;241m600\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     19\u001b[0m     \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdirichlet_alpha\u001b[39m\u001b[38;5;124m'\u001b[39m: \u001b[38;5;241m0.3\u001b[39m\n\u001b[1;32m     20\u001b[0m }\n\u001b[1;32m     22\u001b[0m alphaZero \u001b[38;5;241m=\u001b[39m AlphaZeroParallel(model, optimizer, game, args)\n\u001b[0;32m---> 23\u001b[0m \u001b[43malphaZero\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlearn\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
            "Cell \u001b[0;32mIn[9], line 79\u001b[0m, in \u001b[0;36mAlphaZeroParallel.learn\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     77\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel\u001b[38;5;241m.\u001b[39meval()\n\u001b[1;32m     78\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m selfPlay_iteration \u001b[38;5;129;01min\u001b[39;00m trange(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mnum_selfPlay_iterations\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m/\u001b[39m\u001b[38;5;241m/\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mnum_parallel_games\u001b[39m\u001b[38;5;124m'\u001b[39m]):\n\u001b[0;32m---> 79\u001b[0m     memory \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mselfPlay\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     81\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel\u001b[38;5;241m.\u001b[39mtrain()\n\u001b[1;32m     82\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m epoch \u001b[38;5;129;01min\u001b[39;00m trange(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mnum_epochs\u001b[39m\u001b[38;5;124m'\u001b[39m]):\n",
            "Cell \u001b[0;32mIn[9], line 18\u001b[0m, in \u001b[0;36mAlphaZeroParallel.selfPlay\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     15\u001b[0m states \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mstack([spg\u001b[38;5;241m.\u001b[39mstate \u001b[38;5;28;01mfor\u001b[39;00m spg \u001b[38;5;129;01min\u001b[39;00m spGames])\n\u001b[1;32m     16\u001b[0m neutral_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mchange_perspective(states, player)\n\u001b[0;32m---> 18\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmcts\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msearch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mneutral_states\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mspGames\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     20\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;28mlen\u001b[39m(spGames))[::\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]:\n\u001b[1;32m     21\u001b[0m     spg \u001b[38;5;241m=\u001b[39m spGames[i]\n",
            "File \u001b[0;32m~/anaconda3/envs/myenv/lib/python3.8/site-packages/torch/autograd/grad_mode.py:27\u001b[0m, in \u001b[0;36m_DecoratorContextManager.__call__.<locals>.decorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m     25\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecorate_context\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m     26\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclone():\n\u001b[0;32m---> 27\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
            "Cell \u001b[0;32mIn[8], line 47\u001b[0m, in \u001b[0;36mMCTSParallel.search\u001b[0;34m(self, states, spGames)\u001b[0m\n\u001b[1;32m     44\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(expandable_spGames) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m     45\u001b[0m     states \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mstack([spGames[mappingIdx]\u001b[38;5;241m.\u001b[39mnode\u001b[38;5;241m.\u001b[39mstate \u001b[38;5;28;01mfor\u001b[39;00m mappingIdx \u001b[38;5;129;01min\u001b[39;00m expandable_spGames])\n\u001b[0;32m---> 47\u001b[0m     policy, value \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     48\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtensor\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgame\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_encoded_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstates\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevice\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     49\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     50\u001b[0m     policy \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39msoftmax(policy, axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)\u001b[38;5;241m.\u001b[39mcpu()\u001b[38;5;241m.\u001b[39mnumpy()\n\u001b[1;32m     51\u001b[0m     value \u001b[38;5;241m=\u001b[39m value\u001b[38;5;241m.\u001b[39mcpu()\u001b[38;5;241m.\u001b[39mnumpy()\n",
            "File \u001b[0;32m~/anaconda3/envs/myenv/lib/python3.8/site-packages/torch/nn/modules/module.py:1130\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1126\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1127\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1128\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1129\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1130\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1131\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1132\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
            "Cell \u001b[0;32mIn[4], line 38\u001b[0m, in \u001b[0;36mResNet.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     36\u001b[0m x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstartBlock(x)\n\u001b[1;32m     37\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m resBlock \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbackBone:\n\u001b[0;32m---> 38\u001b[0m     x \u001b[38;5;241m=\u001b[39m \u001b[43mresBlock\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     39\u001b[0m policy \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpolicyHead(x)\n\u001b[1;32m     40\u001b[0m value \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mvalueHead(x)\n",
            "File \u001b[0;32m~/anaconda3/envs/myenv/lib/python3.8/site-packages/torch/nn/modules/module.py:1130\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1126\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1127\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1128\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1129\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1130\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1131\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1132\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
            "Cell \u001b[0;32mIn[4], line 54\u001b[0m, in \u001b[0;36mResBlock.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     52\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, x):\n\u001b[1;32m     53\u001b[0m     residual \u001b[38;5;241m=\u001b[39m x\n\u001b[0;32m---> 54\u001b[0m     x \u001b[38;5;241m=\u001b[39m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrelu\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbn1\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconv1\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     55\u001b[0m     x \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbn2(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconv2(x))\n\u001b[1;32m     56\u001b[0m     x \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m residual\n",
            "File \u001b[0;32m~/anaconda3/envs/myenv/lib/python3.8/site-packages/torch/nn/functional.py:1457\u001b[0m, in \u001b[0;36mrelu\u001b[0;34m(input, inplace)\u001b[0m\n\u001b[1;32m   1455\u001b[0m     result \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mrelu_(\u001b[38;5;28minput\u001b[39m)\n\u001b[1;32m   1456\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1457\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrelu\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1458\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ],
      "source": [
        "game = ConnectFour()\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "model = ResNet(game, 9, 128, device)\n",
        "\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=0.001, weight_decay=0.0001)\n",
        "\n",
        "args = {\n",
        "    'C': 2,\n",
        "    'num_searches': 600,\n",
        "    'num_iterations': 8,\n",
        "    'num_selfPlay_iterations': 500,\n",
        "    'num_parallel_games': 100,\n",
        "    'num_epochs': 4,\n",
        "    'batch_size': 128,\n",
        "    'temperature': 1.25,\n",
        "    'dirichlet_epsilon': 0.25,\n",
        "    'dirichlet_alpha': 0.3\n",
        "}\n",
        "\n",
        "alphaZero = AlphaZeroParallel(model, optimizer, game, args)\n",
        "alphaZero.learn()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "7c470145",
      "metadata": {
        "scrolled": true,
        "id": "7c470145",
        "outputId": "485cb6a0-3478-4690-edca-5642a67cb77c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[[0. 0. 0. 0. 0. 0. 0.]\n",
            " [0. 0. 0. 0. 0. 0. 0.]\n",
            " [0. 0. 0. 0. 0. 0. 0.]\n",
            " [0. 0. 0. 0. 0. 0. 0.]\n",
            " [0. 0. 0. 0. 0. 0. 0.]\n",
            " [0. 0. 0. 0. 0. 0. 0.]]\n",
            "valid_moves [0, 1, 2, 3, 4, 5, 6]\n",
            "1:3\n",
            "[[0. 0. 0. 0. 0. 0. 0.]\n",
            " [0. 0. 0. 0. 0. 0. 0.]\n",
            " [0. 0. 0. 0. 0. 0. 0.]\n",
            " [0. 0. 0. 0. 0. 0. 0.]\n",
            " [0. 0. 0. 0. 0. 0. 0.]\n",
            " [0. 0. 0. 1. 0. 0. 0.]]\n",
            "[[ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0. -1.  0.  0.  0.]\n",
            " [ 0.  0.  0.  1.  0.  0.  0.]]\n",
            "valid_moves [0, 1, 2, 3, 4, 5, 6]\n",
            "1:4\n",
            "[[ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0. -1.  0.  0.  0.]\n",
            " [ 0.  0.  0.  1.  1.  0.  0.]]\n",
            "[[ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0. -1.  0.  0.  0.]\n",
            " [ 0.  0. -1.  1.  1.  0.  0.]]\n",
            "valid_moves [0, 1, 2, 3, 4, 5, 6]\n",
            "1:5\n",
            "[[ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0. -1.  0.  0.  0.]\n",
            " [ 0.  0. -1.  1.  1.  1.  0.]]\n",
            "[[ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0. -1.  0.  0.  0.]\n",
            " [ 0.  0. -1.  1.  1.  1. -1.]]\n",
            "valid_moves [0, 1, 2, 3, 4, 5, 6]\n",
            "1:4\n",
            "[[ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0. -1.  1.  0.  0.]\n",
            " [ 0.  0. -1.  1.  1.  1. -1.]]\n",
            "[[ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0. -1.  0.  0.]\n",
            " [ 0.  0.  0. -1.  1.  0.  0.]\n",
            " [ 0.  0. -1.  1.  1.  1. -1.]]\n",
            "valid_moves [0, 1, 2, 3, 4, 5, 6]\n",
            "1:5\n",
            "[[ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0. -1.  0.  0.]\n",
            " [ 0.  0.  0. -1.  1.  1.  0.]\n",
            " [ 0.  0. -1.  1.  1.  1. -1.]]\n",
            "[[ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0. -1. -1.  0.  0.]\n",
            " [ 0.  0.  0. -1.  1.  1.  0.]\n",
            " [ 0.  0. -1.  1.  1.  1. -1.]]\n",
            "valid_moves [0, 1, 2, 3, 4, 5, 6]\n",
            "1:3\n",
            "[[ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  1.  0.  0.  0.]\n",
            " [ 0.  0.  0. -1. -1.  0.  0.]\n",
            " [ 0.  0.  0. -1.  1.  1.  0.]\n",
            " [ 0.  0. -1.  1.  1.  1. -1.]]\n",
            "[[ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  1. -1.  0.  0.]\n",
            " [ 0.  0.  0. -1. -1.  0.  0.]\n",
            " [ 0.  0.  0. -1.  1.  1.  0.]\n",
            " [ 0.  0. -1.  1.  1.  1. -1.]]\n",
            "valid_moves [0, 1, 2, 3, 4, 5, 6]\n",
            "1:4\n",
            "[[ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  1.  0.  0.]\n",
            " [ 0.  0.  0.  1. -1.  0.  0.]\n",
            " [ 0.  0.  0. -1. -1.  0.  0.]\n",
            " [ 0.  0.  0. -1.  1.  1.  0.]\n",
            " [ 0.  0. -1.  1.  1.  1. -1.]]\n",
            "[[ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  1.  0.  0.]\n",
            " [ 0.  0.  0.  1. -1.  0.  0.]\n",
            " [ 0.  0.  0. -1. -1. -1.  0.]\n",
            " [ 0.  0.  0. -1.  1.  1.  0.]\n",
            " [ 0.  0. -1.  1.  1.  1. -1.]]\n",
            "valid_moves [0, 1, 2, 3, 4, 5, 6]\n",
            "1:5\n",
            "[[ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  1.  0.  0.]\n",
            " [ 0.  0.  0.  1. -1.  1.  0.]\n",
            " [ 0.  0.  0. -1. -1. -1.  0.]\n",
            " [ 0.  0.  0. -1.  1.  1.  0.]\n",
            " [ 0.  0. -1.  1.  1.  1. -1.]]\n",
            "[[ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  1.  0.  0.]\n",
            " [ 0.  0.  0.  1. -1.  1.  0.]\n",
            " [ 0.  0.  0. -1. -1. -1.  0.]\n",
            " [ 0.  0. -1. -1.  1.  1.  0.]\n",
            " [ 0.  0. -1.  1.  1.  1. -1.]]\n",
            "valid_moves [0, 1, 2, 3, 4, 5, 6]\n",
            "1:3\n",
            "[[ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  1.  1.  0.  0.]\n",
            " [ 0.  0.  0.  1. -1.  1.  0.]\n",
            " [ 0.  0.  0. -1. -1. -1.  0.]\n",
            " [ 0.  0. -1. -1.  1.  1.  0.]\n",
            " [ 0.  0. -1.  1.  1.  1. -1.]]\n",
            "[[ 0.  0.  0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0.  1.  1.  0.  0.]\n",
            " [ 0.  0.  0.  1. -1.  1.  0.]\n",
            " [ 0.  0. -1. -1. -1. -1.  0.]\n",
            " [ 0.  0. -1. -1.  1.  1.  0.]\n",
            " [ 0.  0. -1.  1.  1.  1. -1.]]\n",
            "-1 won\n"
          ]
        }
      ],
      "source": [
        "game = ConnectFour()\n",
        "player = 1\n",
        "\n",
        "args = {\n",
        "    'C': 2,\n",
        "    'num_searches': 600,\n",
        "    'dirichlet_epsilon': 0.,\n",
        "    'dirichlet_alpha': 0.3\n",
        "}\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "model = ResNet(game, 9, 128, device)\n",
        "model.load_state_dict(torch.load(\"model_7_ConnectFour.pt\", map_location=device))\n",
        "model.eval()\n",
        "\n",
        "mcts = MCTS(game, args, model)\n",
        "\n",
        "state = game.get_initial_state()\n",
        "\n",
        "\n",
        "while True:\n",
        "    print(state)\n",
        "\n",
        "    if player == 1:\n",
        "        valid_moves = game.get_valid_moves(state)\n",
        "        print(\"valid_moves\", [i for i in range(game.action_size) if valid_moves[i] == 1])\n",
        "        action = int(input(f\"{player}:\"))\n",
        "\n",
        "        if valid_moves[action] == 0:\n",
        "            print(\"action not valid\")\n",
        "            continue\n",
        "\n",
        "    else:\n",
        "        neutral_state = game.change_perspective(state, player)\n",
        "        mcts_probs = mcts.search(neutral_state)\n",
        "        action = np.argmax(mcts_probs)\n",
        "\n",
        "    state = game.get_next_state(state, action, player)\n",
        "\n",
        "    value, is_terminal = game.get_value_and_terminated(state, action)\n",
        "\n",
        "    if is_terminal:\n",
        "        print(state)\n",
        "        if value == 1:\n",
        "            print(player, \"won\")\n",
        "        else:\n",
        "            print(\"draw\")\n",
        "        break\n",
        "\n",
        "    player = game.get_opponent(player)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c3528828",
      "metadata": {
        "id": "c3528828",
        "outputId": "3ddc948e-04fb-4005-981f-4c0cce46f40b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1.9.11\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<iframe srcdoc=\"<!--\n",
              "  Copyright 2020 Kaggle Inc\n",
              "\n",
              "  Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);\n",
              "  you may not use this file except in compliance with the License.\n",
              "  You may obtain a copy of the License at\n",
              "\n",
              "      http://www.apache.org/licenses/LICENSE-2.0\n",
              "\n",
              "  Unless required by applicable law or agreed to in writing, software\n",
              "  distributed under the License is distributed on an &quot;AS IS&quot; BASIS,\n",
              "  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
              "  See the License for the specific language governing permissions and\n",
              "  limitations under the License.\n",
              "-->\n",
              "<!DOCTYPE html>\n",
              "<html lang=&quot;en&quot;>\n",
              "  <head>\n",
              "    <title>Kaggle Simulation Player</title>\n",
              "    <meta name=&quot;viewport&quot; content=&quot;width=device-width,initial-scale=1&quot; />\n",
              "    <link\n",
              "      rel=&quot;stylesheet&quot;\n",
              "      href=&quot;https://cdnjs.cloudflare.com/ajax/libs/meyer-reset/2.0/reset.css&quot;\n",
              "      crossorigin=&quot;anonymous&quot;\n",
              "    />\n",
              "    <style type=&quot;text/css&quot;>\n",
              "      html,\n",
              "      body {\n",
              "        height: 100%;\n",
              "        font-family: sans-serif;\n",
              "        margin: 0px;\n",
              "      }\n",
              "      canvas {\n",
              "        /* image-rendering: -moz-crisp-edges;\n",
              "        image-rendering: -webkit-crisp-edges;\n",
              "        image-rendering: pixelated;\n",
              "        image-rendering: crisp-edges; */\n",
              "      }\n",
              "    </style>\n",
              "    <script src=&quot;https://unpkg.com/preact@10.0.1/dist/preact.umd.js&quot;></script>\n",
              "    <script src=&quot;https://unpkg.com/preact@10.0.1/hooks/dist/hooks.umd.js&quot;></script>\n",
              "    <script src=&quot;https://unpkg.com/htm@2.2.1/dist/htm.umd.js&quot;></script>\n",
              "    <script>\n",
              "      // Polyfill for Styled Components\n",
              "      window.React = {\n",
              "        ...preact,\n",
              "        createElement: preact.h,\n",
              "        PropTypes: { func: {} },\n",
              "      };\n",
              "    </script>\n",
              "    <script src=&quot;https://unpkg.com/styled-components@3.5.0-0/dist/styled-components.min.js&quot;></script>\n",
              "  </head>\n",
              "  <body>\n",
              "    <script>\n",
              "      \n",
              "window.kaggle = {\n",
              "  &quot;debug&quot;: false,\n",
              "  &quot;playing&quot;: true,\n",
              "  &quot;step&quot;: 0,\n",
              "  &quot;controls&quot;: true,\n",
              "  &quot;environment&quot;: {\n",
              "    &quot;id&quot;: &quot;d32c346e-9eed-11ed-b8b1-1d310d57a33b&quot;,\n",
              "    &quot;name&quot;: &quot;tictactoe&quot;,\n",
              "    &quot;title&quot;: &quot;Tic Tac Toe&quot;,\n",
              "    &quot;description&quot;: &quot;Classic Tic Tac Toe&quot;,\n",
              "    &quot;version&quot;: &quot;1.0.0&quot;,\n",
              "    &quot;configuration&quot;: {\n",
              "      &quot;episodeSteps&quot;: 10,\n",
              "      &quot;actTimeout&quot;: 1,\n",
              "      &quot;runTimeout&quot;: 1200,\n",
              "      &quot;agentTimeout&quot;: 2\n",
              "    },\n",
              "    &quot;specification&quot;: {\n",
              "      &quot;action&quot;: {\n",
              "        &quot;description&quot;: &quot;Position to place a mark on the board.&quot;,\n",
              "        &quot;type&quot;: &quot;integer&quot;,\n",
              "        &quot;minimum&quot;: 0,\n",
              "        &quot;maximum&quot;: 8,\n",
              "        &quot;default&quot;: 0\n",
              "      },\n",
              "      &quot;agents&quot;: [\n",
              "        2\n",
              "      ],\n",
              "      &quot;configuration&quot;: {\n",
              "        &quot;episodeSteps&quot;: {\n",
              "          &quot;description&quot;: &quot;Maximum number of steps in the episode.&quot;,\n",
              "          &quot;type&quot;: &quot;integer&quot;,\n",
              "          &quot;minimum&quot;: 1,\n",
              "          &quot;default&quot;: 10\n",
              "        },\n",
              "        &quot;actTimeout&quot;: {\n",
              "          &quot;description&quot;: &quot;Maximum runtime (seconds) to obtain an action from an agent.&quot;,\n",
              "          &quot;type&quot;: &quot;number&quot;,\n",
              "          &quot;minimum&quot;: 0,\n",
              "          &quot;default&quot;: 1\n",
              "        },\n",
              "        &quot;runTimeout&quot;: {\n",
              "          &quot;description&quot;: &quot;Maximum runtime (seconds) of an episode (not necessarily DONE).&quot;,\n",
              "          &quot;type&quot;: &quot;number&quot;,\n",
              "          &quot;minimum&quot;: 0,\n",
              "          &quot;default&quot;: 1200\n",
              "        },\n",
              "        &quot;agentTimeout&quot;: {\n",
              "          &quot;description&quot;: &quot;Obsolete field kept for backwards compatibility, please use observation.remainingOverageTime.&quot;,\n",
              "          &quot;type&quot;: &quot;number&quot;,\n",
              "          &quot;minimum&quot;: 0,\n",
              "          &quot;default&quot;: 2\n",
              "        }\n",
              "      },\n",
              "      &quot;info&quot;: {},\n",
              "      &quot;observation&quot;: {\n",
              "        &quot;remainingOverageTime&quot;: {\n",
              "          &quot;description&quot;: &quot;Total remaining banked time (seconds) that can be used in excess of per-step actTimeouts -- agent is disqualified with TIMEOUT status when this drops below 0.&quot;,\n",
              "          &quot;shared&quot;: false,\n",
              "          &quot;type&quot;: &quot;number&quot;,\n",
              "          &quot;minimum&quot;: 0,\n",
              "          &quot;default&quot;: 2\n",
              "        },\n",
              "        &quot;step&quot;: {\n",
              "          &quot;description&quot;: &quot;Current step within the episode.&quot;,\n",
              "          &quot;type&quot;: &quot;integer&quot;,\n",
              "          &quot;shared&quot;: true,\n",
              "          &quot;minimum&quot;: 0,\n",
              "          &quot;default&quot;: 0\n",
              "        },\n",
              "        &quot;board&quot;: {\n",
              "          &quot;description&quot;: &quot;Serialized 3x3 grid. 0 = Empty, 1 = X, 2 = O&quot;,\n",
              "          &quot;type&quot;: &quot;array&quot;,\n",
              "          &quot;shared&quot;: true,\n",
              "          &quot;default&quot;: [\n",
              "            0,\n",
              "            0,\n",
              "            0,\n",
              "            0,\n",
              "            0,\n",
              "            0,\n",
              "            0,\n",
              "            0,\n",
              "            0\n",
              "          ],\n",
              "          &quot;minItems&quot;: 9,\n",
              "          &quot;maxItems&quot;: 9\n",
              "        },\n",
              "        &quot;mark&quot;: {\n",
              "          &quot;description&quot;: &quot;Mark for the agent to use&quot;,\n",
              "          &quot;defaults&quot;: [\n",
              "            1,\n",
              "            2\n",
              "          ],\n",
              "          &quot;enum&quot;: [\n",
              "            1,\n",
              "            2\n",
              "          ]\n",
              "        }\n",
              "      },\n",
              "      &quot;reward&quot;: {\n",
              "        &quot;description&quot;: &quot;-1 = Lost, 0 = Draw/Ongoing, 1 = Won&quot;,\n",
              "        &quot;enum&quot;: [\n",
              "          -1,\n",
              "          0,\n",
              "          1\n",
              "        ],\n",
              "        &quot;default&quot;: 0,\n",
              "        &quot;type&quot;: [\n",
              "          &quot;number&quot;,\n",
              "          &quot;null&quot;\n",
              "        ]\n",
              "      }\n",
              "    },\n",
              "    &quot;steps&quot;: [\n",
              "      [\n",
              "        {\n",
              "          &quot;action&quot;: 0,\n",
              "          &quot;reward&quot;: 0,\n",
              "          &quot;info&quot;: {},\n",
              "          &quot;observation&quot;: {\n",
              "            &quot;remainingOverageTime&quot;: 2,\n",
              "            &quot;step&quot;: 0,\n",
              "            &quot;board&quot;: [\n",
              "              0,\n",
              "              0,\n",
              "              0,\n",
              "              0,\n",
              "              0,\n",
              "              0,\n",
              "              0,\n",
              "              0,\n",
              "              0\n",
              "            ],\n",
              "            &quot;mark&quot;: 1\n",
              "          },\n",
              "          &quot;status&quot;: &quot;ACTIVE&quot;\n",
              "        },\n",
              "        {\n",
              "          &quot;action&quot;: 0,\n",
              "          &quot;reward&quot;: 0,\n",
              "          &quot;info&quot;: {},\n",
              "          &quot;observation&quot;: {\n",
              "            &quot;remainingOverageTime&quot;: 2,\n",
              "            &quot;mark&quot;: 2\n",
              "          },\n",
              "          &quot;status&quot;: &quot;INACTIVE&quot;\n",
              "        }\n",
              "      ],\n",
              "      [\n",
              "        {\n",
              "          &quot;action&quot;: 4,\n",
              "          &quot;reward&quot;: 0,\n",
              "          &quot;info&quot;: {},\n",
              "          &quot;observation&quot;: {\n",
              "            &quot;remainingOverageTime&quot;: 2,\n",
              "            &quot;step&quot;: 1,\n",
              "            &quot;board&quot;: [\n",
              "              0,\n",
              "              0,\n",
              "              0,\n",
              "              0,\n",
              "              1,\n",
              "              0,\n",
              "              0,\n",
              "              0,\n",
              "              0\n",
              "            ],\n",
              "            &quot;mark&quot;: 1\n",
              "          },\n",
              "          &quot;status&quot;: &quot;INACTIVE&quot;\n",
              "        },\n",
              "        {\n",
              "          &quot;action&quot;: 0,\n",
              "          &quot;reward&quot;: 0,\n",
              "          &quot;info&quot;: {},\n",
              "          &quot;observation&quot;: {\n",
              "            &quot;remainingOverageTime&quot;: 2,\n",
              "            &quot;mark&quot;: 2\n",
              "          },\n",
              "          &quot;status&quot;: &quot;ACTIVE&quot;\n",
              "        }\n",
              "      ],\n",
              "      [\n",
              "        {\n",
              "          &quot;action&quot;: 0,\n",
              "          &quot;reward&quot;: 0,\n",
              "          &quot;info&quot;: {},\n",
              "          &quot;observation&quot;: {\n",
              "            &quot;remainingOverageTime&quot;: 2,\n",
              "            &quot;step&quot;: 2,\n",
              "            &quot;board&quot;: [\n",
              "              0,\n",
              "              0,\n",
              "              2,\n",
              "              0,\n",
              "              1,\n",
              "              0,\n",
              "              0,\n",
              "              0,\n",
              "              0\n",
              "            ],\n",
              "            &quot;mark&quot;: 1\n",
              "          },\n",
              "          &quot;status&quot;: &quot;ACTIVE&quot;\n",
              "        },\n",
              "        {\n",
              "          &quot;action&quot;: 2,\n",
              "          &quot;reward&quot;: 0,\n",
              "          &quot;info&quot;: {},\n",
              "          &quot;observation&quot;: {\n",
              "            &quot;remainingOverageTime&quot;: 2,\n",
              "            &quot;mark&quot;: 2\n",
              "          },\n",
              "          &quot;status&quot;: &quot;INACTIVE&quot;\n",
              "        }\n",
              "      ],\n",
              "      [\n",
              "        {\n",
              "          &quot;action&quot;: 5,\n",
              "          &quot;reward&quot;: 0,\n",
              "          &quot;info&quot;: {},\n",
              "          &quot;observation&quot;: {\n",
              "            &quot;remainingOverageTime&quot;: 2,\n",
              "            &quot;step&quot;: 3,\n",
              "            &quot;board&quot;: [\n",
              "              0,\n",
              "              0,\n",
              "              2,\n",
              "              0,\n",
              "              1,\n",
              "              1,\n",
              "              0,\n",
              "              0,\n",
              "              0\n",
              "            ],\n",
              "            &quot;mark&quot;: 1\n",
              "          },\n",
              "          &quot;status&quot;: &quot;INACTIVE&quot;\n",
              "        },\n",
              "        {\n",
              "          &quot;action&quot;: 0,\n",
              "          &quot;reward&quot;: 0,\n",
              "          &quot;info&quot;: {},\n",
              "          &quot;observation&quot;: {\n",
              "            &quot;remainingOverageTime&quot;: 2,\n",
              "            &quot;mark&quot;: 2\n",
              "          },\n",
              "          &quot;status&quot;: &quot;ACTIVE&quot;\n",
              "        }\n",
              "      ],\n",
              "      [\n",
              "        {\n",
              "          &quot;action&quot;: 0,\n",
              "          &quot;reward&quot;: 0,\n",
              "          &quot;info&quot;: {},\n",
              "          &quot;observation&quot;: {\n",
              "            &quot;remainingOverageTime&quot;: 2,\n",
              "            &quot;step&quot;: 4,\n",
              "            &quot;board&quot;: [\n",
              "              0,\n",
              "              0,\n",
              "              2,\n",
              "              2,\n",
              "              1,\n",
              "              1,\n",
              "              0,\n",
              "              0,\n",
              "              0\n",
              "            ],\n",
              "            &quot;mark&quot;: 1\n",
              "          },\n",
              "          &quot;status&quot;: &quot;ACTIVE&quot;\n",
              "        },\n",
              "        {\n",
              "          &quot;action&quot;: 3,\n",
              "          &quot;reward&quot;: 0,\n",
              "          &quot;info&quot;: {},\n",
              "          &quot;observation&quot;: {\n",
              "            &quot;remainingOverageTime&quot;: 2,\n",
              "            &quot;mark&quot;: 2\n",
              "          },\n",
              "          &quot;status&quot;: &quot;INACTIVE&quot;\n",
              "        }\n",
              "      ],\n",
              "      [\n",
              "        {\n",
              "          &quot;action&quot;: 0,\n",
              "          &quot;reward&quot;: 0,\n",
              "          &quot;info&quot;: {},\n",
              "          &quot;observation&quot;: {\n",
              "            &quot;remainingOverageTime&quot;: 2,\n",
              "            &quot;step&quot;: 5,\n",
              "            &quot;board&quot;: [\n",
              "              1,\n",
              "              0,\n",
              "              2,\n",
              "              2,\n",
              "              1,\n",
              "              1,\n",
              "              0,\n",
              "              0,\n",
              "              0\n",
              "            ],\n",
              "            &quot;mark&quot;: 1\n",
              "          },\n",
              "          &quot;status&quot;: &quot;INACTIVE&quot;\n",
              "        },\n",
              "        {\n",
              "          &quot;action&quot;: 0,\n",
              "          &quot;reward&quot;: 0,\n",
              "          &quot;info&quot;: {},\n",
              "          &quot;observation&quot;: {\n",
              "            &quot;remainingOverageTime&quot;: 2,\n",
              "            &quot;mark&quot;: 2\n",
              "          },\n",
              "          &quot;status&quot;: &quot;ACTIVE&quot;\n",
              "        }\n",
              "      ],\n",
              "      [\n",
              "        {\n",
              "          &quot;action&quot;: 0,\n",
              "          &quot;reward&quot;: 0,\n",
              "          &quot;info&quot;: {},\n",
              "          &quot;observation&quot;: {\n",
              "            &quot;remainingOverageTime&quot;: 2,\n",
              "            &quot;step&quot;: 6,\n",
              "            &quot;board&quot;: [\n",
              "              1,\n",
              "              0,\n",
              "              2,\n",
              "              2,\n",
              "              1,\n",
              "              1,\n",
              "              0,\n",
              "              0,\n",
              "              2\n",
              "            ],\n",
              "            &quot;mark&quot;: 1\n",
              "          },\n",
              "          &quot;status&quot;: &quot;ACTIVE&quot;\n",
              "        },\n",
              "        {\n",
              "          &quot;action&quot;: 8,\n",
              "          &quot;reward&quot;: 0,\n",
              "          &quot;info&quot;: {},\n",
              "          &quot;observation&quot;: {\n",
              "            &quot;remainingOverageTime&quot;: 2,\n",
              "            &quot;mark&quot;: 2\n",
              "          },\n",
              "          &quot;status&quot;: &quot;INACTIVE&quot;\n",
              "        }\n",
              "      ],\n",
              "      [\n",
              "        {\n",
              "          &quot;action&quot;: 6,\n",
              "          &quot;reward&quot;: 0,\n",
              "          &quot;info&quot;: {},\n",
              "          &quot;observation&quot;: {\n",
              "            &quot;remainingOverageTime&quot;: 2,\n",
              "            &quot;step&quot;: 7,\n",
              "            &quot;board&quot;: [\n",
              "              1,\n",
              "              0,\n",
              "              2,\n",
              "              2,\n",
              "              1,\n",
              "              1,\n",
              "              1,\n",
              "              0,\n",
              "              2\n",
              "            ],\n",
              "            &quot;mark&quot;: 1\n",
              "          },\n",
              "          &quot;status&quot;: &quot;INACTIVE&quot;\n",
              "        },\n",
              "        {\n",
              "          &quot;action&quot;: 0,\n",
              "          &quot;reward&quot;: 0,\n",
              "          &quot;info&quot;: {},\n",
              "          &quot;observation&quot;: {\n",
              "            &quot;remainingOverageTime&quot;: 2,\n",
              "            &quot;mark&quot;: 2\n",
              "          },\n",
              "          &quot;status&quot;: &quot;ACTIVE&quot;\n",
              "        }\n",
              "      ],\n",
              "      [\n",
              "        {\n",
              "          &quot;action&quot;: 0,\n",
              "          &quot;reward&quot;: 0,\n",
              "          &quot;info&quot;: {},\n",
              "          &quot;observation&quot;: {\n",
              "            &quot;remainingOverageTime&quot;: 2,\n",
              "            &quot;step&quot;: 8,\n",
              "            &quot;board&quot;: [\n",
              "              1,\n",
              "              2,\n",
              "              2,\n",
              "              2,\n",
              "              1,\n",
              "              1,\n",
              "              1,\n",
              "              0,\n",
              "              2\n",
              "            ],\n",
              "            &quot;mark&quot;: 1\n",
              "          },\n",
              "          &quot;status&quot;: &quot;ACTIVE&quot;\n",
              "        },\n",
              "        {\n",
              "          &quot;action&quot;: 1,\n",
              "          &quot;reward&quot;: 0,\n",
              "          &quot;info&quot;: {},\n",
              "          &quot;observation&quot;: {\n",
              "            &quot;remainingOverageTime&quot;: 2,\n",
              "            &quot;mark&quot;: 2\n",
              "          },\n",
              "          &quot;status&quot;: &quot;INACTIVE&quot;\n",
              "        }\n",
              "      ],\n",
              "      [\n",
              "        {\n",
              "          &quot;action&quot;: 7,\n",
              "          &quot;reward&quot;: 0,\n",
              "          &quot;info&quot;: {},\n",
              "          &quot;observation&quot;: {\n",
              "            &quot;remainingOverageTime&quot;: 2,\n",
              "            &quot;step&quot;: 9,\n",
              "            &quot;board&quot;: [\n",
              "              1,\n",
              "              2,\n",
              "              2,\n",
              "              2,\n",
              "              1,\n",
              "              1,\n",
              "              1,\n",
              "              1,\n",
              "              2\n",
              "            ],\n",
              "            &quot;mark&quot;: 1\n",
              "          },\n",
              "          &quot;status&quot;: &quot;DONE&quot;\n",
              "        },\n",
              "        {\n",
              "          &quot;action&quot;: 0,\n",
              "          &quot;reward&quot;: 0,\n",
              "          &quot;info&quot;: {},\n",
              "          &quot;observation&quot;: {\n",
              "            &quot;remainingOverageTime&quot;: 2,\n",
              "            &quot;mark&quot;: 2\n",
              "          },\n",
              "          &quot;status&quot;: &quot;DONE&quot;\n",
              "        }\n",
              "      ]\n",
              "    ],\n",
              "    &quot;rewards&quot;: [\n",
              "      0,\n",
              "      0\n",
              "    ],\n",
              "    &quot;statuses&quot;: [\n",
              "      &quot;DONE&quot;,\n",
              "      &quot;DONE&quot;\n",
              "    ],\n",
              "    &quot;schema_version&quot;: 1,\n",
              "    &quot;info&quot;: {}\n",
              "  },\n",
              "  &quot;logs&quot;: [\n",
              "    [],\n",
              "    [],\n",
              "    [\n",
              "      {\n",
              "        &quot;duration&quot;: 0.235713,\n",
              "        &quot;stdout&quot;: &quot;&quot;,\n",
              "        &quot;stderr&quot;: &quot;&quot;\n",
              "      },\n",
              "      {}\n",
              "    ],\n",
              "    [\n",
              "      {},\n",
              "      {\n",
              "        &quot;duration&quot;: 0.229516,\n",
              "        &quot;stdout&quot;: &quot;&quot;,\n",
              "        &quot;stderr&quot;: &quot;&quot;\n",
              "      }\n",
              "    ],\n",
              "    [\n",
              "      {\n",
              "        &quot;duration&quot;: 0.192958,\n",
              "        &quot;stdout&quot;: &quot;&quot;,\n",
              "        &quot;stderr&quot;: &quot;&quot;\n",
              "      },\n",
              "      {}\n",
              "    ],\n",
              "    [\n",
              "      {},\n",
              "      {\n",
              "        &quot;duration&quot;: 0.106337,\n",
              "        &quot;stdout&quot;: &quot;&quot;,\n",
              "        &quot;stderr&quot;: &quot;&quot;\n",
              "      }\n",
              "    ],\n",
              "    [\n",
              "      {\n",
              "        &quot;duration&quot;: 0.09835,\n",
              "        &quot;stdout&quot;: &quot;&quot;,\n",
              "        &quot;stderr&quot;: &quot;&quot;\n",
              "      },\n",
              "      {}\n",
              "    ],\n",
              "    [\n",
              "      {},\n",
              "      {\n",
              "        &quot;duration&quot;: 0.036468,\n",
              "        &quot;stdout&quot;: &quot;&quot;,\n",
              "        &quot;stderr&quot;: &quot;&quot;\n",
              "      }\n",
              "    ],\n",
              "    [\n",
              "      {\n",
              "        &quot;duration&quot;: 0.043578,\n",
              "        &quot;stdout&quot;: &quot;&quot;,\n",
              "        &quot;stderr&quot;: &quot;&quot;\n",
              "      },\n",
              "      {}\n",
              "    ],\n",
              "    [\n",
              "      {},\n",
              "      {\n",
              "        &quot;duration&quot;: 0.017656,\n",
              "        &quot;stdout&quot;: &quot;&quot;,\n",
              "        &quot;stderr&quot;: &quot;&quot;\n",
              "      }\n",
              "    ],\n",
              "    [\n",
              "      {\n",
              "        &quot;duration&quot;: 0.010232,\n",
              "        &quot;stdout&quot;: &quot;&quot;,\n",
              "        &quot;stderr&quot;: &quot;&quot;\n",
              "      },\n",
              "      {}\n",
              "    ]\n",
              "  ],\n",
              "  &quot;mode&quot;: &quot;ipython&quot;\n",
              "};\n",
              "\n",
              "\n",
              "window.kaggle.renderer = // Copyright 2020 Kaggle Inc\n",
              "//\n",
              "// Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);\n",
              "// you may not use this file except in compliance with the License.\n",
              "// You may obtain a copy of the License at\n",
              "//\n",
              "//      http://www.apache.org/licenses/LICENSE-2.0\n",
              "//\n",
              "// Unless required by applicable law or agreed to in writing, software\n",
              "// distributed under the License is distributed on an &quot;AS IS&quot; BASIS,\n",
              "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
              "// See the License for the specific language governing permissions and\n",
              "// limitations under the License.\n",
              "\n",
              "async function renderer(context) {\n",
              "  const {\n",
              "    act,\n",
              "    agents,\n",
              "    environment,\n",
              "    frame,\n",
              "    height = 400,\n",
              "    interactive,\n",
              "    isInteractive,\n",
              "    parent,\n",
              "    step,\n",
              "    update,\n",
              "    width = 400,\n",
              "  } = context;\n",
              "\n",
              "  // Common Dimensions.\n",
              "  const canvasSize = Math.min(height, width);\n",
              "  const unit = 8;\n",
              "  const offset = canvasSize > 400 ? canvasSize * 0.1 : unit / 2;\n",
              "  const cellSize = (canvasSize - offset * 2) / 3;\n",
              "\n",
              "  // Canvas Setup.\n",
              "  let canvas = parent.querySelector(&quot;canvas&quot;);\n",
              "  if (!canvas) {\n",
              "    canvas = document.createElement(&quot;canvas&quot;);\n",
              "    parent.appendChild(canvas);\n",
              "\n",
              "    if (interactive) {\n",
              "      canvas.addEventListener(&quot;click&quot;, evt => {\n",
              "        if (!isInteractive()) return;\n",
              "        const rect = evt.target.getBoundingClientRect();\n",
              "        const x = evt.clientX - rect.left - offset;\n",
              "        const y = evt.clientY - rect.top - offset;\n",
              "        act(Math.floor(x / cellSize) + Math.floor(y / cellSize) * 3);\n",
              "      });\n",
              "    }\n",
              "  }\n",
              "  canvas.style.cursor = isInteractive() ? &quot;pointer&quot; : &quot;default&quot;;\n",
              "\n",
              "  // Canvas setup and reset.\n",
              "  let c = canvas.getContext(&quot;2d&quot;);\n",
              "  canvas.width = canvasSize;\n",
              "  canvas.height = canvasSize;\n",
              "  c.clearRect(0, 0, canvas.width, canvas.height);\n",
              "\n",
              "  const drawStyle = ({\n",
              "    lineWidth = 1,\n",
              "    lineCap,\n",
              "    strokeStyle = &quot;#FFF&quot;,\n",
              "    shadow,\n",
              "  }) => {\n",
              "    c.lineWidth = lineWidth;\n",
              "    c.strokeStyle = strokeStyle;\n",
              "    if (lineCap) c.lineCap = lineCap;\n",
              "    if (shadow) {\n",
              "      c.shadowOffsetX = shadow.offsetX || 0;\n",
              "      c.shadowOffsetY = shadow.offsetY || 0;\n",
              "      c.shadowColor = shadow.color || strokeStyle;\n",
              "      c.shadowBlur = shadow.blur || 0;\n",
              "    }\n",
              "  };\n",
              "\n",
              "  const drawLine = ({ x1, y1, x2, y2, style }) => {\n",
              "    c.beginPath();\n",
              "    drawStyle(style || {});\n",
              "    c.moveTo((x1 || 0) + offset, (y1 || 0) + offset);\n",
              "    c.lineTo((x2 || x1) + offset, (y2 || y1) + offset);\n",
              "    c.stroke();\n",
              "  };\n",
              "\n",
              "  const drawArc = ({ x, y, radius, sAngle, eAngle, style }) => {\n",
              "    drawStyle(style || {});\n",
              "    c.beginPath();\n",
              "    c.arc(x, y, radius, sAngle, eAngle);\n",
              "    c.stroke();\n",
              "  };\n",
              "\n",
              "  // Draw the Grid.\n",
              "  const gridFrame = step === 0 ? frame : 1;\n",
              "  const drawGridLine = ({\n",
              "    x1s = 0,\n",
              "    y1s = 0,\n",
              "    x2s,\n",
              "    y2s,\n",
              "    x1o = 0,\n",
              "    x2o = 0,\n",
              "    y1o = 0,\n",
              "    y2o = 0,\n",
              "  }) =>\n",
              "    drawLine({\n",
              "      x1: x1s * cellSize + x1o * unit,\n",
              "      x2: (x2s || x1s) * cellSize + x2o * unit,\n",
              "      y1: y1s * cellSize + y1o * unit,\n",
              "      y2: (y2s || y1s) * cellSize + y2o * unit,\n",
              "      style: { strokeStyle: &quot;#046BBF&quot; },\n",
              "    });\n",
              "\n",
              "  // Vertical.\n",
              "  drawGridLine({ x1s: 1, y1s: 0, y2s: gridFrame, y2o: -1 });\n",
              "  drawGridLine({ x1s: 2, y1s: 0, y2s: gridFrame, y2o: -1 });\n",
              "  drawGridLine({ x1s: 1, y1s: 1, y2s: 1 + gridFrame, y1o: 1, y2o: -1 });\n",
              "  drawGridLine({ x1s: 2, y1s: 1, y2s: 1 + gridFrame, y1o: 1, y2o: -1 });\n",
              "  drawGridLine({ x1s: 1, y1s: 2, y2s: 2 + gridFrame, y1o: 1 });\n",
              "  drawGridLine({ x1s: 2, y1s: 2, y2s: 2 + gridFrame, y1o: 1 });\n",
              "\n",
              "  // Horizontal.\n",
              "  drawGridLine({ x1s: 0, y1s: 1, x2s: gridFrame, x2o: -1 });\n",
              "  drawGridLine({ x1s: 1, y1s: 1, x2s: 1 + gridFrame, x1o: 1, x2o: -1 });\n",
              "  drawGridLine({ x1s: 2, y1s: 1, x2s: 2 + gridFrame, x1o: 1 });\n",
              "  drawGridLine({ x1s: 0, y1s: 2, x2s: gridFrame, x2o: -1 });\n",
              "  drawGridLine({ x1s: 1, y1s: 2, x2s: 1 + gridFrame, x1o: 1, x2o: -1 });\n",
              "  drawGridLine({ x1s: 2, y1s: 2, x2s: 2 + gridFrame, x1o: 1 });\n",
              "\n",
              "  // Draw the Pieces.\n",
              "  const drawX = (cell, cellFrame) => {\n",
              "    const part = cellSize / 4;\n",
              "    const gap = Math.min(Math.sqrt((unit * unit) / 2), canvasSize / 50);\n",
              "    const row = Math.floor(cell / 3);\n",
              "    const col = cell % 3;\n",
              "\n",
              "    const drawXLine = ({ x1, y1, x2, y2 }) =>\n",
              "      drawLine({\n",
              "        x1: col * cellSize + x1,\n",
              "        y1: row * cellSize + y1,\n",
              "        x2: col * cellSize + x2,\n",
              "        y2: row * cellSize + y2,\n",
              "        style: {\n",
              "          strokeStyle: &quot;#00FFFF&quot;,\n",
              "          lineWidth: 2,\n",
              "          lineCap: &quot;round&quot;,\n",
              "          shadow: { blur: 8 },\n",
              "        },\n",
              "      });\n",
              "\n",
              "    drawXLine({\n",
              "      x1: part,\n",
              "      y1: part,\n",
              "      x2: part + part * 2 * cellFrame,\n",
              "      y2: part + part * 2 * cellFrame,\n",
              "    });\n",
              "    if (Math.round(cellFrame) === 1) {\n",
              "      drawXLine({\n",
              "        x1: part,\n",
              "        y1: part * 3,\n",
              "        x2: part * 2 - gap,\n",
              "        y2: part * 2 + gap,\n",
              "      });\n",
              "      drawXLine({\n",
              "        x1: part * 2 + gap,\n",
              "        y1: part * 2 - gap,\n",
              "        x2: part * 3,\n",
              "        y2: part,\n",
              "      });\n",
              "    }\n",
              "  };\n",
              "  const drawO = (cell, cellFrame) => {\n",
              "    const row = Math.floor(cell / 3);\n",
              "    const col = cell % 3;\n",
              "    const radius = cellSize / 4 + 1; // +1 is for optical illusion.\n",
              "    const gap =\n",
              "      (Math.acos((2 * (radius ^ 2) - (unit ^ 2)) / (2 * radius * radius)) /\n",
              "        180) *\n",
              "      Math.PI *\n",
              "      unit;\n",
              "    const x = cellSize * col + cellSize / 2 + offset;\n",
              "    const y = cellSize * row + cellSize / 2 + offset;\n",
              "\n",
              "    const drawOArc = (sAngle, eAngle) =>\n",
              "      drawArc({\n",
              "        x,\n",
              "        y,\n",
              "        radius,\n",
              "        sAngle,\n",
              "        eAngle,\n",
              "        style: {\n",
              "          lineWidth: 2,\n",
              "          strokeStyle: &quot;#FFF&quot;,\n",
              "          shadow: { blur: 8 },\n",
              "        },\n",
              "      });\n",
              "\n",
              "    drawOArc(\n",
              "      -Math.PI / 2 + gap,\n",
              "      -Math.PI / 2 + gap + (Math.PI - gap * 2) * cellFrame\n",
              "    );\n",
              "    drawOArc(\n",
              "      Math.PI / 2 + gap,\n",
              "      Math.PI / 2 + gap + (Math.PI - gap * 2) * cellFrame\n",
              "    );\n",
              "  };\n",
              "\n",
              "  const board = environment.steps[step][0].observation.board;\n",
              "\n",
              "  board.forEach((value, cell) => {\n",
              "    const cellFrame =\n",
              "      step <= 1 ||\n",
              "      environment.steps[step - 1][0].observation.board[cell] !== value\n",
              "        ? frame\n",
              "        : 1;\n",
              "    if (value === 1) drawX(cell, cellFrame);\n",
              "    if (value === 2) drawO(cell, cellFrame);\n",
              "  });\n",
              "\n",
              "  // Draw the winning line.\n",
              "  // [cell1, cell2, cell3, x1, y1, x2, y2]\n",
              "  const checks = [\n",
              "    [0, 1, 2, 1 / 19, 1 / 6, 18 / 19, 1 / 6],\n",
              "    [3, 4, 5, 1 / 19, 1 / 2, 18 / 19, 1 / 2],\n",
              "    [6, 7, 8, 1 / 19, 5 / 6, 18 / 19, 5 / 6],\n",
              "    [0, 3, 6, 1 / 6, 1 / 19, 1 / 6, 18 / 19],\n",
              "    [1, 4, 7, 1 / 2, 1 / 19, 1 / 2, 18 / 19],\n",
              "    [2, 5, 8, 5 / 6, 1 / 19, 5 / 6, 18 / 19],\n",
              "    [0, 4, 8, 1 / 19, 1 / 19, 18 / 19, 18 / 19],\n",
              "    [2, 4, 6, 18 / 19, 1 / 19, 1 / 19, 18 / 19],\n",
              "  ];\n",
              "  for (const check of checks) {\n",
              "    if (\n",
              "      board[check[0]] !== 0 &&\n",
              "      board[check[0]] === board[check[1]] &&\n",
              "      board[check[0]] === board[check[2]]\n",
              "    ) {\n",
              "      const x1 = check[3] * (cellSize * 3);\n",
              "      const y1 = check[4] * (cellSize * 3);\n",
              "      const winFrame = frame < 0.5 ? 0 : (frame - 0.5) / 0.5;\n",
              "      if (winFrame > 0) {\n",
              "        drawLine({\n",
              "          x1,\n",
              "          y1,\n",
              "          x2: x1 + (check[5] * (cellSize * 3) - x1) * winFrame,\n",
              "          y2: y1 + (check[6] * (cellSize * 3) - y1) * winFrame,\n",
              "          style: {\n",
              "            strokeStyle: &quot;#FFF&quot;,\n",
              "            lineWidth: 3 * winFrame,\n",
              "            shadow: { blur: 8 * winFrame },\n",
              "          },\n",
              "        });\n",
              "      }\n",
              "      break;\n",
              "    }\n",
              "  }\n",
              "\n",
              "  // Upgrade the legend.\n",
              "  if (agents.length && (!agents[0].color || !agents[0].image)) {\n",
              "    const getPieceImage = drawFn => {\n",
              "      const pieceCanvas = document.createElement(&quot;canvas&quot;);\n",
              "      parent.appendChild(pieceCanvas);\n",
              "      pieceCanvas.style.marginLeft = &quot;10000px&quot;;\n",
              "      pieceCanvas.width = cellSize + offset * 2;\n",
              "      pieceCanvas.height = cellSize + offset * 2;\n",
              "      c = pieceCanvas.getContext(&quot;2d&quot;);\n",
              "      drawFn(0, 1);\n",
              "      const dataUrl = pieceCanvas.toDataURL();\n",
              "      parent.removeChild(pieceCanvas);\n",
              "      return dataUrl;\n",
              "    };\n",
              "\n",
              "    agents.forEach(agent => {\n",
              "      agent.color = agent.index === 0 ? &quot;#0FF&quot; : &quot;#FFF&quot;;\n",
              "      agent.image = getPieceImage(agent.index === 0 ? drawX : drawO);\n",
              "    });\n",
              "    update({ agents });\n",
              "  }\n",
              "};\n",
              "\n",
              "\n",
              "    \n",
              "    </script>\n",
              "    <script>\n",
              "      const h = htm.bind(preact.h);\n",
              "      const { useContext, useEffect, useRef, useState } = preactHooks;\n",
              "      const styled = window.styled.default;\n",
              "\n",
              "      const Context = preact.createContext({});\n",
              "\n",
              "      const Loading = styled.div`\n",
              "        animation: rotate360 1.1s infinite linear;\n",
              "        border: 8px solid rgba(255, 255, 255, 0.2);\n",
              "        border-left-color: #0cb1ed;\n",
              "        border-radius: 50%;\n",
              "        height: 40px;\n",
              "        position: relative;\n",
              "        transform: translateZ(0);\n",
              "        width: 40px;\n",
              "\n",
              "        @keyframes rotate360 {\n",
              "          0% {\n",
              "            transform: rotate(0deg);\n",
              "          }\n",
              "          100% {\n",
              "            transform: rotate(360deg);\n",
              "          }\n",
              "        }\n",
              "      `;\n",
              "\n",
              "      const Logo = styled(\n",
              "        (props) => h`\n",
              "        <a href=&quot;https://kaggle.com&quot; target=&quot;_blank&quot; className=${props.className}>\n",
              "          <svg width=&quot;62px&quot; height=&quot;20px&quot; viewBox=&quot;0 0 62 24&quot; version=&quot;1.1&quot; xmlns=&quot;http://www.w3.org/2000/svg&quot;>\n",
              "            <g fill=&quot;#1EBEFF&quot; fill-rule=&quot;nonzero&quot;>\n",
              "              <path d=&quot;M10.2,17.8c0,0.1-0.1,0.1-0.2,0.1H7.7c-0.1,0-0.3-0.1-0.4-0.2l-3.8-4.9l-1.1,1v3.8 c0,0.2-0.1,0.3-0.3,0.3H0.3c-0.2,0-0.3-0.1-0.3-0.3V0.3C0.1,0.1,0.2,0,0.3,0h1.8c0.2,0,0.3,0.1,0.3,0.3V11L7,6.3 c0.1-0.1,0.2-0.2,0.4-0.2h2.4c0.1,0,0.2,0,0.2,0.1c0,0.1,0,0.2,0,0.2l-4.9,4.7l5.1,6.3C10.2,17.6,10.2,17.7,10.2,17.8z&quot;/>\n",
              "              <path d=&quot;M19.6,17.9h-1.8c-0.2,0-0.3-0.1-0.3-0.3v-0.4c-0.8,0.6-1.8,0.9-3,0.9c-1.1,0-2-0.3-2.8-1 c-0.8-0.7-1.2-1.6-1.2-2.7c0-1.7,1.1-2.9,3.2-3.5c0.8-0.2,2.1-0.5,3.8-0.6c0.1-0.6-0.1-1.2-0.5-1.7c-0.4-0.5-1-0.7-1.7-0.7 c-1,0-2,0.4-3,1C12.2,9.1,12.1,9.1,12,9l-0.9-1.3C11,7.5,11,7.4,11.1,7.3c1.3-0.9,2.7-1.4,4.2-1.4c1.1,0,2.1,0.3,2.8,0.8 c1.1,0.8,1.7,2,1.7,3.7v7.3C19.9,17.8,19.8,17.9,19.6,17.9z M17.5,12.4c-1.7,0.2-2.9,0.4-3.5,0.7c-0.9,0.4-1.2,0.9-1.1,1.6 c0.1,0.4,0.2,0.7,0.6,0.9c0.3,0.2,0.7,0.4,1.1,0.4c1.2,0.1,2.2-0.2,2.9-1V12.4z&quot;/>\n",
              "              <path d=&quot;M30.6,22.5c-0.9,1-2.3,1.5-4,1.5c-1,0-2-0.3-2.9-0.8c-0.2-0.1-0.4-0.3-0.7-0.5 c-0.3-0.2-0.6-0.5-0.9-0.7c-0.1-0.1-0.1-0.2,0-0.4l1.2-1.2c0.1-0.1,0.1-0.1,0.2-0.1c0.1,0,0.1,0,0.2,0.1c1,1,1.9,1.5,2.8,1.5 c2.1,0,3.2-1.1,3.2-3.3v-1.4c-0.8,0.7-1.9,1-3.3,1c-1.7,0-3-0.6-4-1.9c-0.8-1.1-1.3-2.5-1.3-4.2c0-1.6,0.4-3,1.2-4.1 c0.9-1.3,2.3-2,4-2c1.3,0,2.4,0.3,3.3,1V6.4c0-0.2,0.1-0.3,0.3-0.3h1.8c0.2,0,0.3,0.1,0.3,0.3v11.7C32,20,31.5,21.5,30.6,22.5z M29.7,9.9c-0.4-1.1-1.4-1.7-3-1.7c-2,0-3.1,1.3-3.1,3.8c0,1.4,0.3,2.4,1,3.1c0.5,0.5,1.2,0.8,2,0.8c1.6,0,2.7-0.6,3.1-1.7V9.9z&quot;/>\n",
              "              <path d=&quot;M42.9,22.5c-0.9,1-2.3,1.5-4,1.5c-1,0-2-0.3-2.9-0.8c-0.2-0.1-0.4-0.3-0.7-0.5 c-0.3-0.2-0.6-0.5-0.9-0.7c-0.1-0.1-0.1-0.2,0-0.4l1.2-1.2c0.1-0.1,0.1-0.1,0.2-0.1c0.1,0,0.1,0,0.2,0.1c1,1,1.9,1.5,2.8,1.5 c2.1,0,3.2-1.1,3.2-3.3v-1.4c-0.8,0.7-1.9,1-3.3,1c-1.7,0-3-0.6-4-1.9c-0.8-1.1-1.3-2.5-1.3-4.2c0-1.6,0.4-3,1.2-4.1 c0.9-1.3,2.3-2,4-2c1.3,0,2.4,0.3,3.3,1V6.4c0-0.2,0.1-0.3,0.3-0.3H44c0.2,0,0.3,0.1,0.3,0.3v11.7C44.3,20,43.8,21.5,42.9,22.5z M42,9.9c-0.4-1.1-1.4-1.7-3-1.7c-2,0-3.1,1.3-3.1,3.8c0,1.4,0.3,2.4,1,3.1c0.5,0.5,1.2,0.8,2,0.8c1.6,0,2.7-0.6,3.1-1.7L42,9.9 L42,9.9z&quot;/>\n",
              "              <path d=&quot;M48.3,17.9h-1.8c-0.2,0-0.3-0.1-0.3-0.3V0.3c0-0.2,0.1-0.3,0.3-0.3h1.8c0.2,0,0.3,0.1,0.3,0.3 v17.3C48.5,17.8,48.5,17.9,48.3,17.9z&quot;/>\n",
              "              <path d=&quot;M61.4,12.6c0,0.2-0.1,0.3-0.3,0.3h-8.5c0.1,0.9,0.5,1.6,1.1,2.2c0.7,0.6,1.6,0.9,2.7,0.9 c1,0,1.8-0.3,2.6-0.8c0.2-0.1,0.3-0.1,0.4,0l1.2,1.3c0.1,0.1,0.1,0.3,0,0.4c-1.3,0.9-2.7,1.4-4.4,1.4c-1.8,0-3.3-0.6-4.4-1.8 c-1.1-1.2-1.7-2.7-1.7-4.5c0-1.7,0.6-3.2,1.7-4.4c1-1.1,2.4-1.6,4.1-1.6c1.6,0,2.9,0.6,4,1.7c1.1,1.2,1.6,2.6,1.5,4.4L61.4,12.6 z M58,8.7c-0.6-0.5-1.3-0.8-2.1-0.8c-0.8,0-1.5,0.3-2.1,0.8c-0.6,0.5-1,1.2-1.1,2H59C59,9.9,58.6,9.3,58,8.7z&quot;/>\n",
              "            </g>\n",
              "          </svg>\n",
              "        </a>\n",
              "      `\n",
              "      )`\n",
              "        display: inline-flex;\n",
              "      `;\n",
              "\n",
              "      const Header = styled((props) => {\n",
              "        const { environment } = useContext(Context);\n",
              "\n",
              "        return h`<div className=${props.className} >\n",
              "          <${Logo} />\n",
              "          <span><b>Left / Right Arrow:</b> Increase / Decrease Step</span><span><b>0-9 Row Keys:</b> Playback Speed</span><span><b>Space:</b> Pause / Play</span>\n",
              "          ${environment.title}\n",
              "        </div>`;\n",
              "      })`\n",
              "        align-items: center;\n",
              "        border-bottom: 4px solid #212121;\n",
              "        box-sizing: border-box;\n",
              "        color: #fff;\n",
              "        display: flex;\n",
              "        flex: 0 0 36px;\n",
              "        font-size: 14px;\n",
              "        justify-content: space-between;\n",
              "        padding: 0 8px;\n",
              "        width: 100%;\n",
              "      `;\n",
              "\n",
              "      const Renderer = styled((props) => {\n",
              "        const context = useContext(Context);\n",
              "        const { animate, debug, playing, renderer, speed } = context;\n",
              "        const ref = preact.createRef();\n",
              "\n",
              "        useEffect(async () => {\n",
              "          if (!ref.current) return;\n",
              "\n",
              "          const renderFrame = async (start, step, lastFrame) => {\n",
              "            if (step !== context.step) return;\n",
              "            if (lastFrame === 1) {\n",
              "              if (!animate) return;\n",
              "              start = Date.now();\n",
              "            }\n",
              "            const frame =\n",
              "              playing || animate\n",
              "                ? Math.min((Date.now() - start) / speed, 1)\n",
              "                : 1;\n",
              "            try {\n",
              "              if (debug) console.time(&quot;render&quot;);\n",
              "              await renderer({\n",
              "                ...context,\n",
              "                frame,\n",
              "                height: ref.current.clientHeight,\n",
              "                hooks: preactHooks,\n",
              "                parent: ref.current,\n",
              "                preact,\n",
              "                styled,\n",
              "                width: ref.current.clientWidth,\n",
              "              });\n",
              "            } catch (error) {\n",
              "              if (debug) console.error(error);\n",
              "              console.log({ ...context, frame, error });\n",
              "            } finally {\n",
              "              if (debug) console.timeEnd(&quot;render&quot;);\n",
              "            }\n",
              "            window.requestAnimationFrame(() => renderFrame(start, step, frame));\n",
              "          };\n",
              "\n",
              "          await renderFrame(Date.now(), context.step);\n",
              "        }, [ref.current, context.step, context.renderer]);\n",
              "\n",
              "        return h`<div className=${props.className} ref=${ref} />`;\n",
              "      })`\n",
              "        align-items: center;\n",
              "        box-sizing: border-box;\n",
              "        display: flex;\n",
              "        height: 100%;\n",
              "        left: 0;\n",
              "        justify-content: center;\n",
              "        position: absolute;\n",
              "        top: 0;\n",
              "        width: 100%;\n",
              "      `;\n",
              "\n",
              "      const Processing = styled((props) => {\n",
              "        const { processing } = useContext(Context);\n",
              "        const text = processing === true ? &quot;Processing...&quot; : processing;\n",
              "        return h`<div className=${props.className}>${text}</div>`;\n",
              "      })`\n",
              "        bottom: 0;\n",
              "        color: #fff;\n",
              "        font-size: 12px;\n",
              "        left: 0;\n",
              "        line-height: 24px;\n",
              "        position: absolute;\n",
              "        text-align: center;\n",
              "        width: 100%;\n",
              "      `;\n",
              "\n",
              "      const Viewer = styled((props) => {\n",
              "        const { processing } = useContext(Context);\n",
              "        return h`<div className=${props.className}>\n",
              "          <${Renderer} />\n",
              "          ${processing && h`<${Processing} />`}\n",
              "        </div>`;\n",
              "      })`\n",
              "        background-color: #000b2a;\n",
              "        background-image: radial-gradient(\n",
              "          circle closest-side,\n",
              "          #000b49,\n",
              "          #000b2a\n",
              "        );\n",
              "        display: flex;\n",
              "        flex: 1;\n",
              "        overflow: hidden;\n",
              "        position: relative;\n",
              "        width: 100%;\n",
              "      `;\n",
              "\n",
              "      // Partitions the elements of arr into subarrays of max length num.\n",
              "      const groupIntoSets = (arr, num) => {\n",
              "        const sets = [];\n",
              "        arr.forEach(a => {\n",
              "          if (sets.length === 0 || sets[sets.length - 1].length === num) {\n",
              "            sets.push([]);\n",
              "          }\n",
              "          sets[sets.length - 1].push(a);\n",
              "        });\n",
              "        return sets;\n",
              "      }\n",
              "\n",
              "      // Expects `width` input prop to set proper max-width for agent name span.\n",
              "      const Legend = styled((props) => {\n",
              "        const { agents, legend } = useContext(Context);\n",
              "\n",
              "        const agentPairs = groupIntoSets(agents.sort((a, b) => a.index - b.index), 2);\n",
              "\n",
              "        return h`<div className=${props.className}>\n",
              "          ${agentPairs.map(agentList =>\n",
              "            h`<ul>\n",
              "                ${agentList.map(a =>\n",
              "                  h`<li key=${a.id} title=&quot;id: ${a.id}&quot; style=&quot;color:${a.color || &quot;#FFF&quot;}&quot;>\n",
              "                      ${a.image && h`<img src=${a.image} />`}\n",
              "                      <span>${a.name}</span>\n",
              "                    </li>`\n",
              "                )}\n",
              "              </ul>`)}\n",
              "        </div>`;\n",
              "      })`\n",
              "        background-color: #000b2a;\n",
              "        font-family: sans-serif;\n",
              "        font-size: 14px;\n",
              "        height: 48px;\n",
              "        width: 100%;\n",
              "\n",
              "        ul {\n",
              "          align-items: center;\n",
              "          display: flex;\n",
              "          flex-direction: row;\n",
              "          justify-content: center;\n",
              "        }\n",
              "\n",
              "        li {\n",
              "          align-items: center;\n",
              "          display: inline-flex;\n",
              "          transition: color 1s;\n",
              "        }\n",
              "\n",
              "        span {\n",
              "          max-width: ${p => (p.width || 400) * 0.5 - 36}px;\n",
              "          overflow: hidden;\n",
              "          text-overflow: ellipsis;\n",
              "          white-space: nowrap;\n",
              "        }\n",
              "\n",
              "        img {\n",
              "          height: 24px;\n",
              "          margin-left: 4px;\n",
              "          margin-right: 4px;\n",
              "          width: 24px;\n",
              "        }\n",
              "      `;\n",
              "\n",
              "      const StepInput = styled.input.attrs({\n",
              "        type: &quot;range&quot;,\n",
              "      })`\n",
              "        appearance: none;\n",
              "        background: rgba(255, 255, 255, 0.15);\n",
              "        border-radius: 2px;\n",
              "        display: block;\n",
              "        flex: 1;\n",
              "        height: 4px;\n",
              "        opacity: 0.8;\n",
              "        outline: none;\n",
              "        transition: opacity 0.2s;\n",
              "        width: 100%;\n",
              "\n",
              "        &:hover {\n",
              "          opacity: 1;\n",
              "        }\n",
              "\n",
              "        &::-webkit-slider-thumb {\n",
              "          appearance: none;\n",
              "          background: #1ebeff;\n",
              "          border-radius: 100%;\n",
              "          cursor: pointer;\n",
              "          height: 12px;\n",
              "          margin: 0;\n",
              "          position: relative;\n",
              "          width: 12px;\n",
              "\n",
              "          &::after {\n",
              "            content: &quot;&quot;;\n",
              "            position: absolute;\n",
              "            top: 0px;\n",
              "            left: 0px;\n",
              "            width: 200px;\n",
              "            height: 8px;\n",
              "            background: green;\n",
              "          }\n",
              "        }\n",
              "      `;\n",
              "\n",
              "      const PlayButton = styled.button`\n",
              "        align-items: center;\n",
              "        background: none;\n",
              "        border: none;\n",
              "        color: white;\n",
              "        cursor: pointer;\n",
              "        display: flex;\n",
              "        flex: 0 0 56px;\n",
              "        font-size: 20px;\n",
              "        height: 40px;\n",
              "        justify-content: center;\n",
              "        opacity: 0.8;\n",
              "        outline: none;\n",
              "        transition: opacity 0.2s;\n",
              "\n",
              "        &:hover {\n",
              "          opacity: 1;\n",
              "        }\n",
              "      `;\n",
              "\n",
              "      const StepCount = styled.span`\n",
              "        align-items: center;\n",
              "        color: white;\n",
              "        display: flex;\n",
              "        font-size: 14px;\n",
              "        justify-content: center;\n",
              "        opacity: 0.8;\n",
              "        padding: 0 16px;\n",
              "        pointer-events: none;\n",
              "      `;\n",
              "\n",
              "      const Controls = styled((props) => {\n",
              "        const { environment, pause, play, playing, setStep, step } = useContext(\n",
              "          Context\n",
              "        );\n",
              "        const value = step + 1;\n",
              "        const onClick = () => (playing ? pause() : play());\n",
              "        const onInput = (e) => {\n",
              "          pause();\n",
              "          setStep(parseInt(e.target.value) - 1);\n",
              "        };\n",
              "\n",
              "        return h`\n",
              "          <div className=${props.className}>\n",
              "            <${PlayButton} onClick=${onClick}><svg xmlns=&quot;http://www.w3.org/2000/svg&quot; width=&quot;24px&quot; height=&quot;24px&quot; viewBox=&quot;0 0 24 24&quot; fill=&quot;#FFFFFF&quot;>${\n",
              "          playing\n",
              "            ? h`<path d=&quot;M6 19h4V5H6v14zm8-14v14h4V5h-4z&quot;/><path d=&quot;M0 0h24v24H0z&quot; fill=&quot;none&quot;/>`\n",
              "            : h`<path d=&quot;M8 5v14l11-7z&quot;/><path d=&quot;M0 0h24v24H0z&quot; fill=&quot;none&quot;/>`\n",
              "        }</svg><//>\n",
              "            <${StepInput} min=&quot;1&quot; max=${\n",
              "          environment.steps.length\n",
              "        } value=&quot;${value}&quot; onInput=${onInput} />\n",
              "            <${StepCount}>${value} / ${environment.steps.length}<//>\n",
              "          </div>\n",
              "        `;\n",
              "      })`\n",
              "        align-items: center;\n",
              "        border-top: 4px solid #212121;\n",
              "        display: flex;\n",
              "        flex: 0 0 44px;\n",
              "        width: 100%;\n",
              "      `;\n",
              "\n",
              "      const Info = styled((props) => {\n",
              "        const {\n",
              "          environment,\n",
              "          playing,\n",
              "          step,\n",
              "          speed,\n",
              "          animate,\n",
              "          header,\n",
              "          controls,\n",
              "          settings,\n",
              "        } = useContext(Context);\n",
              "\n",
              "        return h`\n",
              "          <div className=${props.className}>\n",
              "            info:\n",
              "            step(${step}),\n",
              "            playing(${playing ? &quot;T&quot; : &quot;F&quot;}),\n",
              "            speed(${speed}),\n",
              "            animate(${animate ? &quot;T&quot; : &quot;F&quot;})\n",
              "          </div>`;\n",
              "      })`\n",
              "        color: #888;\n",
              "        font-family: monospace;\n",
              "        font-size: 12px;\n",
              "      `;\n",
              "\n",
              "      const Settings = styled((props) => {\n",
              "        const { environment, pause, play, playing, setStep, step } = useContext(\n",
              "          Context\n",
              "        );\n",
              "\n",
              "        return h`\n",
              "          <div className=${props.className}>\n",
              "            <${Info} />\n",
              "          </div>\n",
              "        `;\n",
              "      })`\n",
              "        background: #fff;\n",
              "        border-top: 4px solid #212121;\n",
              "        box-sizing: border-box;\n",
              "        padding: 20px;\n",
              "        width: 100%;\n",
              "\n",
              "        h1 {\n",
              "          font-size: 20px;\n",
              "        }\n",
              "      `;\n",
              "\n",
              "      const Player = styled((props) => {\n",
              "        const context = useContext(Context);\n",
              "        const { agents, controls, header, legend, loading, settings, width } = context;\n",
              "        return h`\n",
              "          <div className=${props.className}>\n",
              "            ${loading && h`<${Loading} />`}\n",
              "            ${!loading && header && h`<${Header} />`}\n",
              "            ${!loading && h`<${Viewer} />`}\n",
              "            ${!loading && legend && h`<${Legend} width=${width}/>`}\n",
              "            ${!loading && controls && h`<${Controls} />`}\n",
              "            ${!loading && settings && h`<${Settings} />`}\n",
              "          </div>`;\n",
              "      })`\n",
              "        align-items: center;\n",
              "        background: #212121;\n",
              "        border: 4px solid #212121;\n",
              "        box-sizing: border-box;\n",
              "        display: flex;\n",
              "        flex-direction: column;\n",
              "        height: 100%;\n",
              "        justify-content: center;\n",
              "        position: relative;\n",
              "        width: 100%;\n",
              "      `;\n",
              "\n",
              "      const App = () => {\n",
              "        const renderCountRef = useRef(0);\n",
              "        const [_, setRenderCount] = useState(0);\n",
              "\n",
              "        // These are bindings to the 0-9 keys and are milliseconds of timeout per step\n",
              "        const speeds = [\n",
              "          0,\n",
              "          3000,\n",
              "          1000,\n",
              "          500,\n",
              "          333, // Default\n",
              "          200,\n",
              "          100,\n",
              "          50,\n",
              "          25,\n",
              "          10,\n",
              "        ];\n",
              "\n",
              "        const contextRef = useRef({\n",
              "          animate: false,\n",
              "          agents: [],\n",
              "          controls: false,\n",
              "          debug: false,\n",
              "          environment: { steps: [], info: {} },\n",
              "          header: window.innerHeight >= 600,\n",
              "          height: window.innerHeight,\n",
              "          interactive: false,\n",
              "          legend: true,\n",
              "          loading: false,\n",
              "          playing: false,\n",
              "          processing: false,\n",
              "          renderer: () => &quot;DNE&quot;,\n",
              "          settings: false,\n",
              "          speed: speeds[4],\n",
              "          step: 0,\n",
              "          width: window.innerWidth,\n",
              "        });\n",
              "\n",
              "        // Context helpers.\n",
              "        const rerender = (contextRef.current.rerender = () =>\n",
              "          setRenderCount((renderCountRef.current += 1)));\n",
              "        const setStep = (contextRef.current.setStep = (newStep) => {\n",
              "          contextRef.current.step = newStep;\n",
              "          rerender();\n",
              "        });\n",
              "        const setPlaying = (contextRef.current.setPlaying = (playing) => {\n",
              "          contextRef.current.playing = playing;\n",
              "          rerender();\n",
              "        });\n",
              "        const pause = (contextRef.current.pause = () => setPlaying(false));\n",
              "\n",
              "        const playNext = () => {\n",
              "          const context = contextRef.current;\n",
              "\n",
              "          if (\n",
              "            context.playing &&\n",
              "            context.step < context.environment.steps.length - 1\n",
              "          ) {\n",
              "            setStep(context.step + 1);\n",
              "            play(true);\n",
              "          } else {\n",
              "            pause();\n",
              "          }\n",
              "        };\n",
              "\n",
              "        const play = (contextRef.current.play = (continuing) => {\n",
              "          const context = contextRef.current;\n",
              "          if (context.playing && !continuing) return;\n",
              "          if (!context.playing) setPlaying(true);\n",
              "          if (\n",
              "            !continuing &&\n",
              "            context.step === context.environment.steps.length - 1\n",
              "          ) {\n",
              "            setStep(0);\n",
              "          }\n",
              "          setTimeout(playNext, context.speed);\n",
              "        });\n",
              "\n",
              "        const updateContext = (o) => {\n",
              "          const context = contextRef.current;\n",
              "          Object.assign(context, o, {\n",
              "            environment: { ...context.environment, ...(o.environment || {}) },\n",
              "          });\n",
              "          rerender();\n",
              "        };\n",
              "\n",
              "        // First time setup.\n",
              "        useEffect(() => {\n",
              "          // Timeout is used to ensure useEffect renders once.\n",
              "          setTimeout(() => {\n",
              "            // Initialize context with window.kaggle.\n",
              "            updateContext(window.kaggle || {});\n",
              "\n",
              "            if (window.kaggle.playing) {\n",
              "                play(true);\n",
              "            }\n",
              "\n",
              "            // Listen for messages received to update the context.\n",
              "            window.addEventListener(\n",
              "              &quot;message&quot;,\n",
              "              (event) => {\n",
              "                // Ensure the environment names match before updating.\n",
              "                try {\n",
              "                  if (\n",
              "                    event.data.environment.name ==\n",
              "                    contextRef.current.environment.name\n",
              "                  ) {\n",
              "                    updateContext(event.data);\n",
              "                  }\n",
              "                } catch {}\n",
              "              },\n",
              "              false\n",
              "            );\n",
              "            // Listen for keyboard commands.\n",
              "            window.addEventListener(\n",
              "              &quot;keydown&quot;,\n",
              "              (event) => {\n",
              "                const {\n",
              "                  interactive,\n",
              "                  isInteractive,\n",
              "                  playing,\n",
              "                  step,\n",
              "                  environment,\n",
              "                } = contextRef.current;\n",
              "                const key = event.keyCode;\n",
              "                const zero_key = 48\n",
              "                const nine_key = 57\n",
              "                if (\n",
              "                  interactive ||\n",
              "                  isInteractive() ||\n",
              "                  (key !== 32 && key !== 37 && key !== 39 && !(key >= zero_key && key <= nine_key))\n",
              "                )\n",
              "                  return;\n",
              "\n",
              "                if (key === 32) {\n",
              "                  playing ? pause() : play();\n",
              "                } else if (key === 39) {\n",
              "                  contextRef.current.playing = false;\n",
              "                  if (step < environment.steps.length - 1) setStep(step + 1);\n",
              "                  rerender();\n",
              "                } else if (key === 37) {\n",
              "                  contextRef.current.playing = false;\n",
              "                  if (step > 0) setStep(step - 1);\n",
              "                  rerender();\n",
              "                } else if (key >= zero_key && key <= nine_key) {\n",
              "                  contextRef.current.speed = speeds[key - zero_key];\n",
              "                }\n",
              "                event.preventDefault();\n",
              "                return false;\n",
              "              },\n",
              "              false\n",
              "            );\n",
              "          }, 1);\n",
              "        }, []);\n",
              "\n",
              "        if (contextRef.current.debug) {\n",
              "          console.log(&quot;context&quot;, contextRef.current);\n",
              "        }\n",
              "\n",
              "        // Ability to update context.\n",
              "        contextRef.current.update = updateContext;\n",
              "\n",
              "        // Ability to communicate with ipython.\n",
              "        const execute = (contextRef.current.execute = (source) =>\n",
              "          new Promise((resolve, reject) => {\n",
              "            try {\n",
              "              window.parent.IPython.notebook.kernel.execute(source, {\n",
              "                iopub: {\n",
              "                  output: (resp) => {\n",
              "                    const type = resp.msg_type;\n",
              "                    if (type === &quot;stream&quot;) return resolve(resp.content.text);\n",
              "                    if (type === &quot;error&quot;) return reject(new Error(resp.evalue));\n",
              "                    return reject(new Error(&quot;Unknown message type: &quot; + type));\n",
              "                  },\n",
              "                },\n",
              "              });\n",
              "            } catch (e) {\n",
              "              reject(new Error(&quot;IPython Unavailable: &quot; + e));\n",
              "            }\n",
              "          }));\n",
              "\n",
              "        // Ability to return an action from an interactive session.\n",
              "        contextRef.current.act = (action) => {\n",
              "          const id = contextRef.current.environment.id;\n",
              "          updateContext({ processing: true });\n",
              "          execute(`\n",
              "            import json\n",
              "            from kaggle_environments import interactives\n",
              "            if &quot;${id}&quot; in interactives:\n",
              "                action = json.loads('${JSON.stringify(action)}')\n",
              "                env, trainer = interactives[&quot;${id}&quot;]\n",
              "                trainer.step(action)\n",
              "                print(json.dumps(env.steps))`)\n",
              "            .then((resp) => {\n",
              "              try {\n",
              "                updateContext({\n",
              "                  processing: false,\n",
              "                  environment: { steps: JSON.parse(resp) },\n",
              "                });\n",
              "                play();\n",
              "              } catch (e) {\n",
              "                updateContext({ processing: resp.split(&quot;\\n&quot;)[0] });\n",
              "                console.error(resp, e);\n",
              "              }\n",
              "            })\n",
              "            .catch((e) => console.error(e));\n",
              "        };\n",
              "\n",
              "        // Check if currently interactive.\n",
              "        contextRef.current.isInteractive = () => {\n",
              "          const context = contextRef.current;\n",
              "          const steps = context.environment.steps;\n",
              "          return (\n",
              "            context.interactive &&\n",
              "            !context.processing &&\n",
              "            context.step === steps.length - 1 &&\n",
              "            steps[context.step].some((s) => s.status === &quot;ACTIVE&quot;)\n",
              "          );\n",
              "        };\n",
              "\n",
              "        return h`\n",
              "          <${Context.Provider} value=${contextRef.current}>\n",
              "            <${Player} />\n",
              "          <//>`;\n",
              "      };\n",
              "\n",
              "      preact.render(h`<${App} />`, document.body);\n",
              "    </script>\n",
              "  </body>\n",
              "</html>\n",
              "\" width=\"300\" height=\"300\" frameborder=\"0\"></iframe> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "import kaggle_environments\n",
        "print(kaggle_environments.__version__)\n",
        "\n",
        "class KaggleAgent:\n",
        "    def __init__(self, model, game, args):\n",
        "        self.model = model\n",
        "        self.game = game\n",
        "        self.args = args\n",
        "        if self.args['search']:\n",
        "            self.mcts = MCTS(self.game, self.args, self.model)\n",
        "\n",
        "    def run(self, obs, conf):\n",
        "        player = obs['mark'] if obs['mark'] == 1 else -1\n",
        "        state = np.array(obs['board']).reshape(self.game.row_count, self.game.column_count)\n",
        "        state[state==2] = -1\n",
        "\n",
        "        state = self.game.change_perspective(state, player)\n",
        "\n",
        "        if self.args['search']:\n",
        "            policy = self.mcts.search(state)\n",
        "\n",
        "        else:\n",
        "            policy, _ = self.model.predict(state, augment=self.args['augment']) # Not working with the video's implementation\n",
        "\n",
        "        valid_moves = self.game.get_valid_moves(state)\n",
        "        policy *= valid_moves\n",
        "        policy /= np.sum(policy)\n",
        "\n",
        "        if self.args['temperature'] == 0:\n",
        "            action = int(np.argmax(policy))\n",
        "        elif self.args['temperature'] == float('inf'):\n",
        "            action = np.random.choice([r for r in range(self.game.action_size) if policy[r] > 0])\n",
        "        else:\n",
        "            policy = policy ** (1 / self.args['temperature'])\n",
        "            policy /= np.sum(policy)\n",
        "            action = np.random.choice(self.game.action_size, p=policy)\n",
        "\n",
        "        return action\n",
        "\n",
        "game = TicTacToe()\n",
        "\n",
        "args = {\n",
        "    'C': 2,\n",
        "    'num_searches': 100,\n",
        "    'dirichlet_epsilon': 0.1,\n",
        "    'dirichlet_alpha': 0.3,\n",
        "    'search': True,\n",
        "    'temperature': 0,\n",
        "}\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "model = ResNet(game, 4, 64, device)\n",
        "model.load_state_dict(torch.load(\"model_2.pt\", map_location=device))\n",
        "model.eval()\n",
        "\n",
        "env = kaggle_environments.make(\"tictactoe\")\n",
        "\n",
        "player1 = KaggleAgent(model, game, args)\n",
        "player2 = KaggleAgent(model, game, args)\n",
        "\n",
        "players = [player1.run, player2.run]\n",
        "\n",
        "env.run(players)\n",
        "\n",
        "env.render(mode=\"ipython\")"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "myenv",
      "language": "python",
      "name": "myenv"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.13"
    },
    "vscode": {
      "interpreter": {
        "hash": "2177f1ca12c1330a133c1d40b46100b268ab447cddcbdfdc0c7b2b7e4840e700"
      }
    },
    "colab": {
      "provenance": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}