AlphaZero2_0.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "2geRXAeZlfys",
        "outputId": "6f6c5582-1540-46dd-e8ba-660a09821529"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'1.23.5'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 1
        }
      ],
      "source": [
        "import numpy as np\n",
        "np.__version__"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#TICTACTOE"
      ],
      "metadata": {
        "id": "Fgjb5onDhdgB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class TicTacToe:\n",
        "  def __init__(self):\n",
        "    self.row_count = 3\n",
        "    self.col_count = 3\n",
        "    self.action_size = self.row_count * self.col_count\n",
        "\n",
        "  def get_initial_state(self):\n",
        "    return np.zeros((self.row_count, self.col_count))\n",
        "\n",
        "  def get_next_state(self, state, action, player):\n",
        "    row = action // self.col_count\n",
        "    column = action % self.col_count\n",
        "    state[row, column] = player\n",
        "    return state\n",
        "\n",
        "  def get_valid_moves(self, state):\n",
        "    return (state.reshape(-1) == 0).astype(np.uint8)\n",
        "\n",
        "  def check_win(self, state, action):\n",
        "    row = action // self.col_count\n",
        "    column = action % self.col_count\n",
        "    player = state[row, column]\n",
        "    return (\n",
        "        np.sum(state[row, :]) == player * self.col_count\n",
        "        or np.sum(state[: , column]) == player * self.row_count\n",
        "        or np.sum(np.diag(state)) == player * self.row_count\n",
        "        or np.sum(np.diag(np.flip(state, axis = 0))) == player * self.row_count\n",
        "    )\n",
        "\n",
        "  def get_value_and_terminated(self, state, action):\n",
        "    if self.check_win(state, action) :\n",
        "      return 1, True\n",
        "    if np.sum(self.get_valid_moves(state)) == 0:\n",
        "      return 0, True\n",
        "    return 0, False\n",
        "\n",
        "  def get_opponent(self, player):\n",
        "    return -player\n",
        "\n",
        "  def get_opponent_value(self, value):\n",
        "    return -value\n",
        "\n",
        "  def change_perspective(self, state, player):\n",
        "    return state * player\n"
      ],
      "metadata": {
        "id": "B4Z0CwyllmVm"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tictactoe = TicTacToe()"
      ],
      "metadata": {
        "id": "ntzThsRMpetk"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "player = 1\n",
        "state = tictactoe.get_initial_state()\n",
        "\n",
        "while True:\n",
        "  print(state)\n",
        "\n",
        "  valid_moves = tictactoe.get_valid_moves(state)\n",
        "  print(\"Valid moves : \", [i for i in range(tictactoe.action_size) if valid_moves[i] == 1])\n",
        "  action = int(input(f\"{player} : \"))\n",
        "\n",
        "  if valid_moves[action] == 0:\n",
        "    print(\"action not valid\")\n",
        "    continue\n",
        "\n",
        "  state = tictactoe.get_next_state(state, action, player)\n",
        "\n",
        "  value, is_terminal = tictactoe.get_value_and_terminated(state, action)\n",
        "\n",
        "  if is_terminal:\n",
        "    print(state)\n",
        "    if value == 1:\n",
        "      print(player, \"won\")\n",
        "    else:\n",
        "      print(\"draw\")\n",
        "    break\n",
        "  player = tictactoe.get_opponent(player)"
      ],
      "metadata": {
        "id": "CfRO3S5bpiTN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#MCTS"
      ],
      "metadata": {
        "id": "tWLMkDXehg5P"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class Node:\n",
        "    def __init__(self, game, args, state, parent=None, action_taken=None):\n",
        "        self.game = game\n",
        "        self.args = args\n",
        "        self.state = state\n",
        "        self.parent = parent\n",
        "        self.action_taken = action_taken\n",
        "\n",
        "        self.children = []\n",
        "        self.expandable_moves = game.get_valid_moves(state)\n",
        "\n",
        "        self.visit_count = 0\n",
        "        self.value_sum = 0\n",
        "\n",
        "    def is_fully_expanded(self):\n",
        "        return np.sum(self.expandable_moves) == 0 and len(self.children) > 0\n",
        "\n",
        "    def select(self):\n",
        "        best_child = None\n",
        "        best_ucb = -np.inf\n",
        "\n",
        "        for child in self.children:\n",
        "            ucb = self.get_ucb(child)\n",
        "            if ucb > best_ucb:\n",
        "                best_child = child\n",
        "                best_ucb = ucb\n",
        "\n",
        "        return best_child\n",
        "\n",
        "    def get_ucb(self, child):\n",
        "        q_value = 1 - ((child.value_sum / child.visit_count) + 1) / 2\n",
        "        return q_value + self.args['C'] * math.sqrt(math.log(self.visit_count) / child.visit_count)\n",
        "\n",
        "    def expand(self):\n",
        "        action = np.random.choice(np.where(self.expandable_moves == 1)[0])\n",
        "        self.expandable_moves[action] = 0\n",
        "\n",
        "        child_state = self.state.copy()\n",
        "        child_state = self.game.get_next_state(child_state, action, 1)\n",
        "        child_state = self.game.change_perspective(child_state, player=-1)\n",
        "\n",
        "        child = Node(self.game, self.args, child_state, self, action)\n",
        "        self.children.append(child)\n",
        "        return child\n",
        "\n",
        "    def simulate(self):\n",
        "        value, is_terminal = self.game.get_value_and_terminated(self.state, self.action_taken)\n",
        "        value = self.game.get_opponent_value(value)\n",
        "\n",
        "        if is_terminal:\n",
        "            return value\n",
        "\n",
        "        rollout_state = self.state.copy()\n",
        "        rollout_player = 1\n",
        "        while True:\n",
        "            valid_moves = self.game.get_valid_moves(rollout_state)\n",
        "            action = np.random.choice(np.where(valid_moves == 1)[0])\n",
        "            rollout_state = self.game.get_next_state(rollout_state, action, rollout_player)\n",
        "            value, is_terminal = self.game.get_value_and_terminated(rollout_state, action)\n",
        "            if is_terminal:\n",
        "                if rollout_player == -1:\n",
        "                    value = self.game.get_opponent_value(value)\n",
        "                return value\n",
        "\n",
        "            rollout_player = self.game.get_opponent(rollout_player)\n",
        "\n",
        "    def backpropagate(self, value):\n",
        "        self.value_sum += value\n",
        "        self.visit_count += 1\n",
        "\n",
        "        value = self.game.get_opponent_value(value)\n",
        "        if self.parent is not None:\n",
        "            self.parent.backpropagate(value)\n",
        "\n",
        "\n",
        "class MCTS:\n",
        "    def __init__(self, game, args):\n",
        "        self.game = game\n",
        "        self.args = args\n",
        "\n",
        "    def search(self, state):\n",
        "        root = Node(self.game, self.args, state)\n",
        "\n",
        "        for search in range(self.args['num_searches']):\n",
        "            node = root\n",
        "\n",
        "            while node.is_fully_expanded():\n",
        "                node = node.select()\n",
        "\n",
        "            value, is_terminal = self.game.get_value_and_terminated(node.state, node.action_taken)\n",
        "            value = self.game.get_opponent_value(value)\n",
        "\n",
        "            if not is_terminal:\n",
        "                node = node.expand()\n",
        "                value = node.simulate()\n",
        "\n",
        "            node.backpropagate(value)\n",
        "\n",
        "\n",
        "        action_probs = np.zeros(self.game.action_size)\n",
        "        for child in root.children:\n",
        "            action_probs[child.action_taken] = child.visit_count\n",
        "        action_probs /= np.sum(action_probs)\n",
        "        return action_probs"
      ],
      "metadata": {
        "id": "dlFWivZYqhxz"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tictactoe = TicTacToe()\n",
        "player = 1\n",
        "\n",
        "args = {\n",
        "    'C': 1.41,\n",
        "    'num_searches': 1000\n",
        "}\n",
        "\n",
        "mcts = MCTS(tictactoe, args)\n",
        "\n",
        "state = tictactoe.get_initial_state()\n",
        "\n",
        "while True:\n",
        "    print(state)\n",
        "\n",
        "    if player == 1:\n",
        "        valid_moves = tictactoe.get_valid_moves(state)\n",
        "        print(\"valid_moves\", [i for i in range(tictactoe.action_size) if valid_moves[i] == 1])\n",
        "        action = int(input(f\"{player}:\"))\n",
        "\n",
        "        if valid_moves[action] == 0:\n",
        "            print(\"action not valid\")\n",
        "            continue\n",
        "\n",
        "    else:\n",
        "        neutral_state = tictactoe.change_perspective(state, player)\n",
        "        mcts_probs = mcts.search(neutral_state)\n",
        "\n",
        "        if mcts_probs is None:\n",
        "            print(\"MCTS failed to find a valid action. Stopping.\")\n",
        "            break\n",
        "\n",
        "        action = np.argmax(mcts_probs)\n",
        "        action = tictactoe.change_perspective(action, -player)  # Change the action back to the original perspective\n",
        "\n",
        "    state = tictactoe.get_next_state(state, action, player)\n",
        "\n",
        "    value, is_terminal = tictactoe.get_value_and_terminated(state, action)\n",
        "\n",
        "    if is_terminal:\n",
        "        print(state)\n",
        "        if value == 1:\n",
        "            print(player, \"won\")\n",
        "        else:\n",
        "            print(\"draw\")\n",
        "        break\n",
        "\n",
        "    player = tictactoe.get_opponent(player)\n"
      ],
      "metadata": {
        "id": "fjQ1mwLWgxLT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "vZSgdZ_7jjD0"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}