Frozen Lake Comparison Q and Double Q Learnings Deterministic.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Applying Tabular Methods"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Comparing Q Learning And Double Q Learning - Deterministic"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import Statements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym as gym\n",
    "from gym import spaces\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.offsetbox import OffsetImage, AnnotationBbox\n",
    "import numpy as np\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Frozen Lake Environment\n",
    "\n",
    "Description: The agent is on a frozen lake and must navigate to the goal while avoiding holes and collecting Gems."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FrozenLakeEnv(gym.Env):\n",
    "    metadata = {'render.modes': []}\n",
    "    def __init__(self, gamma=0.9, alpha=0.1, max_timestamp=10):\n",
    "        self.gamma = gamma\n",
    "        self.alpha = alpha\n",
    "\n",
    "        self.obs_space = spaces.Discrete(16)\n",
    "        self.action_space = spaces.Discrete(4)\n",
    "        self.max_timestamp = max_timestamp\n",
    "        self.timestep = 0\n",
    "\n",
    "        ## STATES SET #############\n",
    "        self.state = np.zeros((4, 4))\n",
    "        self.myskater = np.asarray([0, 0])\n",
    "        self.goal_loc = np.asarray([3, 3])\n",
    "        self.gem_loc = [np.asarray([0, 2]), np.asarray([3, 2])]\n",
    "        self.hole_loc = [np.asarray([1, 3]), np.asarray([2, 0])]\n",
    "\n",
    "        self.state[tuple(self.myskater)] = 0.2\n",
    "        self.state[tuple(self.goal_loc)] = 0.8\n",
    "        for pos in self.gem_loc:\n",
    "            self.state[tuple(pos)] = 0.5\n",
    "        for pos in self.hole_loc:\n",
    "            self.state[tuple(pos)] = 0.4\n",
    "        ##########################\n",
    "        \n",
    "        self.prev_state = np.zeros((4, 4))\n",
    "        self.prev_action = None\n",
    "        self.penalty_counter = 0  \n",
    "        self.flag_out_grid = 0  \n",
    "\n",
    "\n",
    "    def step(self, action):\n",
    "        self.prev_state = np.copy(self.state)  \n",
    "        self.prev_action = action  \n",
    "        self.flag_out_grid = 0  \n",
    "\n",
    "        ## ACTIONS SET #############\n",
    "        if action == 0:    # moves right\n",
    "            self.myskater[0] += 1\n",
    "        elif action == 1:   # moves left\n",
    "            self.myskater[0] -= 1\n",
    "        elif action == 2:   # moves up\n",
    "            self.myskater[1] += 1 \n",
    "        elif action == 3:    # moves down\n",
    "            self.myskater[1] -= 1\n",
    "        ##########################\n",
    "            \n",
    "        self.myskater = np.clip(self.myskater, 0, 3)\n",
    "\n",
    "        # If the agent is in the same position as the previous step, choose a different action\n",
    "        prev_state_positions = np.argwhere(self.prev_state == 0.2)\n",
    "        if len(prev_state_positions) > 0 and np.array_equal(self.myskater, prev_state_positions[0]):\n",
    "            while action == self.prev_action:\n",
    "                action = self.action_space.sample()\n",
    "            self.flag_out_grid = 1\n",
    "\n",
    "        self.state = np.zeros((4, 4))\n",
    "        self.state[tuple(self.myskater)] = 0.2\n",
    "        self.state[tuple(self.goal_loc)] = 0.8\n",
    "        for pos in self.gem_loc:\n",
    "            self.state[tuple(pos)] = 0.5\n",
    "        for pos in self.hole_loc:\n",
    "            self.state[tuple(pos)] = 0.4\n",
    "\n",
    "        obs = self.state.flatten()\n",
    "        reward = self.calculate_reward()\n",
    "        penalty = any(np.array_equal(self.myskater, pos) for pos in self.hole_loc)\n",
    "        if penalty:\n",
    "            self.penalty_counter += 1 \n",
    "        self.timestep += 1\n",
    "\n",
    "        terminated = True if np.array_equal(self.myskater, self.goal_loc) else self.timestep >= self.max_timestamp\n",
    "        truncated = True if np.any((self.myskater < 0) | (self.myskater > 3)) else False\n",
    "        if truncated:\n",
    "            self.flag_out_grid = 1  \n",
    "\n",
    "        info = {}\n",
    "\n",
    "        return self.state.flatten(), reward, terminated, truncated, info\n",
    "\n",
    "    def reset(self, **kwargs):\n",
    "        self.state = np.zeros((4, 4))\n",
    "        self.myskater = np.asarray([0, 0])\n",
    "        self.state[tuple(self.myskater)] = 0.2\n",
    "        self.state[tuple(self.goal_loc)] = 0.8\n",
    "\n",
    "        for pos in self.gem_loc:\n",
    "            self.state[tuple(pos)] = 0.5\n",
    "        for pos in self.hole_loc:\n",
    "            self.state[tuple(pos)] = 0.4\n",
    "\n",
    "        self.prev_state = np.zeros((4, 4))\n",
    "        self.prev_action = None\n",
    "        self.flag_out_grid = 0 \n",
    "        \n",
    "        obs = self.state.flatten()\n",
    "        self.timestep = 0\n",
    "        info = {}\n",
    "        self.penalty_counter = 0\n",
    "        return obs, info\n",
    "\n",
    "    def calculate_reward(self):\n",
    "        prev_myskateritions = np.argwhere(self.prev_state == 0.2)\n",
    "        if prev_myskateritions.size == 0:\n",
    "            prev_myskaterition = self.myskater\n",
    "        else:\n",
    "            prev_myskaterition = prev_myskateritions[0]\n",
    "\n",
    "        # Calculating distance to goal before and after the step\n",
    "        prev_distance_to_goal = np.linalg.norm(self.goal_loc - prev_myskaterition)\n",
    "        current_distance_to_goal = np.linalg.norm(self.goal_loc - self.myskater)\n",
    "\n",
    "\n",
    "        ## REWARDS SET #############\n",
    "        if np.array_equal(self.myskater, self.goal_loc):\n",
    "            reward = 10  # Positive reward for reaching goal\n",
    "        elif np.array_equal(self.myskater, self.hole_loc[0]):\n",
    "            reward = -5  # negative reward for reaching holes 1\n",
    "        elif np.array_equal(self.myskater, self.hole_loc[1]):\n",
    "            reward = -6   # negative reward for reaching holes 2\n",
    "        elif np.array_equal(self.myskater, self.gem_loc[0]):\n",
    "            reward = 5   # positive reward for reaching gems 1\n",
    "        elif np.array_equal(self.myskater, self.gem_loc[1]):\n",
    "            reward = 6  # positive reward for reaching gems 2\n",
    "        elif current_distance_to_goal < prev_distance_to_goal:\n",
    "            reward = 1  # Positive reward for moving closer to goal\n",
    "        elif current_distance_to_goal > prev_distance_to_goal:\n",
    "            reward = -1  # Negative reward for moving away to goal\n",
    "        else:\n",
    "            reward = -0.1  # Slight negative reward for no change\n",
    "        ##########################\n",
    "        \n",
    "        return reward\n",
    "    \n",
    "    def get_penalty_count(self):\n",
    "        return self.penalty_counter\n",
    "\n",
    "    def render(self):\n",
    "        fig, ax = plt.subplots()\n",
    "        plt.title('Frozen Lake Environment')\n",
    "\n",
    "        # Load and display the background image\n",
    "        background_img = plt.imread('images/frozen_lake.jpg')\n",
    "        ax.imshow(background_img, extent=(-0.5, 3.5, -0.5, 3.5), origin='upper')\n",
    "\n",
    "       \n",
    "        skater_img = plt.imread('images/icons8-skateboard-100.png')\n",
    "        hole_img = plt.imread('images/icons8-hole-100.png')\n",
    "        gem_img = plt.imread('images/icons8-gems-100.png')\n",
    "        goal_img = plt.imread('images/icons8-flag-100.png')\n",
    "        skater_hole_drown_img = plt.imread('images/agent_hole_drown.png')\n",
    "        skater_gem_lottery_img = plt.imread('images/agent_gems_lottery.png')\n",
    "        agent_flag_winner_img = plt.imread('images/agent_flag_winner.png')\n",
    "        agent_grid_cross_img = plt.imread('images/agent_grid_cross.png')\n",
    "\n",
    "        # Plot Skater\n",
    "        myskater = self.myskater\n",
    "        if self.flag_out_grid:\n",
    "            skater_img = agent_grid_cross_img\n",
    "        agent_box = AnnotationBbox(OffsetImage(skater_img, zoom=0.4), myskater, frameon=False)\n",
    "        ax.add_artist(agent_box)\n",
    "\n",
    "        # Plot Holes\n",
    "        for hole_loc in self.hole_loc:\n",
    "            hole_loc = hole_loc\n",
    "            if np.array_equal(self.myskater, hole_loc):\n",
    "                hole_img = skater_hole_drown_img\n",
    "            else:\n",
    "                hole_img = plt.imread('images/icons8-hole-100.png')\n",
    "            rock_box = AnnotationBbox(OffsetImage(hole_img, zoom=0.4), hole_loc, frameon=False)\n",
    "            ax.add_artist(rock_box)\n",
    "\n",
    "        # Plot Gems\n",
    "        for gem_loc in self.gem_loc:\n",
    "            gem_loc = gem_loc\n",
    "            if np.array_equal(self.myskater, gem_loc):\n",
    "                gem_img = skater_gem_lottery_img\n",
    "            else:\n",
    "                gem_img = plt.imread('images/icons8-gems-100.png')\n",
    "            battery_box = AnnotationBbox(OffsetImage(gem_img, zoom=0.4), gem_loc, frameon=False)\n",
    "            ax.add_artist(battery_box)\n",
    "\n",
    "        # Plot goal\n",
    "        goal_loc = self.goal_loc\n",
    "        goal_loc = self.goal_loc\n",
    "        if np.array_equal(self.myskater, goal_loc):\n",
    "            goal_img = agent_flag_winner_img\n",
    "        else:\n",
    "            goal_img = plt.imread('images/icons8-flag-100.png')\n",
    "        goal_box = AnnotationBbox(OffsetImage(goal_img, zoom=0.4), goal_loc, frameon=False)\n",
    "        ax.add_artist(goal_box)\n",
    "\n",
    "        plt.xticks(np.arange(-0.5, 4.5, 1))\n",
    "        plt.yticks(np.arange(-0.5, 4.5, 1))\n",
    "        plt.gca().set_xticklabels(np.arange(-0.5, 4.5, 1))\n",
    "        plt.gca().set_yticklabels(np.arange(-0.5, 4.5, 1))\n",
    "        plt.show()\n",
    "\n",
    "    def obs_space_to_index(self, obs):\n",
    "        myskater = np.argwhere(obs.reshape(4, 4) == 0.2)\n",
    "        if myskater.size == 0:\n",
    "            return 0 \n",
    "        return myskater[0, 0] * 4 + myskater[0, 1]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 100\n",
      "Q-table:\n",
      "[[ 1.42723997  6.78999302  2.68193215  4.43591888]\n",
      " [ 2.16578485  3.54087052  8.75969117  0.87362594]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [-1.63683016  0.22204377  1.00913149  6.96256755]\n",
      " [-2.77629507  0.62622993  2.11969003 -0.7488315 ]\n",
      " [ 0.80695219  1.65217441  2.99716171 -0.93977439]\n",
      " [ 1.09276051  7.67247227 -2.60672768 -0.19942972]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.27226932 -0.5518115   1.29397071 -3.69723977]\n",
      " [ 1.99804948 -0.32304096  2.27838089 -0.48436502]\n",
      " [ 6.34819007 -1.11210861  1.39558805  0.02939428]\n",
      " [-0.2681253  -1.96242421  0.63446175  0.08252013]\n",
      " [-0.12551501 -0.06515363  0.         -0.81139608]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.        ]]\n",
      "Average Penalties in Last 100 Episodes: 0.82\n",
      "Episode: 100, Average Steps: 9.87\n",
      "Episode: 200\n",
      "Q-table:\n",
      "[[ 7.76114247 14.75020672 10.29057532  9.41702406]\n",
      " [ 8.25202923  7.00407673 14.318215    7.86753844]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 1.35529533  7.92267886  7.71827863 14.77420634]\n",
      " [-1.23055949  5.1462758   8.57466945  1.66018347]\n",
      " [ 1.02030019 11.67040558  3.810906    0.02750318]\n",
      " [ 1.97149531 14.19346777  1.00367498  2.72217745]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.27226932 -0.5518115   1.71231574 -3.69723977]\n",
      " [ 1.99804948  0.99650451  4.53845252 -0.31770528]\n",
      " [10.23998695 -1.00222599  1.39558805  0.38675183]\n",
      " [-0.15249571 -1.96242421  0.68929249  0.08252013]\n",
      " [-0.12551501 -0.06515363  2.01870384 -0.81139608]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.        ]]\n",
      "Average Penalties in Last 100 Episodes: 0.38\n",
      "Episode: 200, Average Steps: 9.95\n",
      "Episode: 300\n",
      "Q-table:\n",
      "[[11.72990371 17.57163019 22.31972985 16.38415418]\n",
      " [11.80562931 13.56446015 24.23251788 11.94251339]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 7.47338295 17.82802859 17.36951117 24.29601776]\n",
      " [ 2.01645929  9.46511197 10.18029736  4.02458681]\n",
      " [ 0.91088794 14.40912846  5.43496824  2.88880912]\n",
      " [ 3.91711126 19.07928347  1.00367498  2.72217745]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.27226932 -0.5518115   3.04409633 -2.21027338]\n",
      " [ 1.99804948  3.11979613  8.3910041  -0.31770528]\n",
      " [16.7191283  -1.00222599  1.39558805  1.45297177]\n",
      " [-0.15249571 -1.96242421  0.68929249  0.08252013]\n",
      " [-0.12551501 -0.06515363  2.01870384 -0.81139608]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.        ]]\n",
      "Average Penalties in Last 100 Episodes: 0.25\n",
      "Episode: 300, Average Steps: 9.96\n",
      "Episode: 400\n",
      "Q-table:\n",
      "[[19.08163981 25.26514522 29.05382365 24.88959976]\n",
      " [16.23877087 20.76757171 31.07429527 16.68371581]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [15.69939879 22.81457104 25.98643935 31.52129309]\n",
      " [ 2.01645929  9.46511197 15.07275018  5.85208184]\n",
      " [ 0.91088794 23.14122492  8.23219173  2.88880912]\n",
      " [ 3.91711126 26.23963573  1.00367498  2.72217745]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.27226932 -0.5518115   3.04409633 -2.21027338]\n",
      " [ 1.99804948  3.11979613  8.3910041  -0.31770528]\n",
      " [16.7191283  -1.00222599  1.39558805  1.45297177]\n",
      " [-0.15249571 -1.96242421  0.68929249  0.08252013]\n",
      " [-0.12551501 -0.06515363  2.01870384 -0.81139608]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.        ]]\n",
      "Average Penalties in Last 100 Episodes: 0.28\n",
      "Episode: 400, Average Steps: 10.0\n",
      "Episode: 500\n",
      "Q-table:\n",
      "[[28.0535013  30.95033359 28.87726968 29.97831582]\n",
      " [17.76113626 27.37436546 38.33654742 19.03959187]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [20.81854895 31.79428961 27.92280058 37.76245159]\n",
      " [ 2.01645929  9.46511197 17.8680711   5.85208184]\n",
      " [ 1.35803848 28.12321075  8.23219173  4.63761884]\n",
      " [ 3.91711126 36.31921542  1.00367498  2.72217745]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.27226932 -0.5518115   3.93319997 -2.21027338]\n",
      " [ 1.99804948  3.11979613 10.03760369 -0.31770528]\n",
      " [21.86636748 -1.00222599  1.39558805  1.45297177]\n",
      " [-0.15249571 -1.96242421  0.68929249  0.08252013]\n",
      " [-0.12551501 -0.06515363  2.01870384 -0.81139608]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.        ]]\n",
      "Average Penalties in Last 100 Episodes: 0.17\n",
      "Episode: 500, Average Steps: 9.93\n",
      "Episode: 600\n",
      "Q-table:\n",
      "[[26.20356258 33.69642231 40.32572392 33.58290496]\n",
      " [17.76113626 27.37436546 42.94805753 21.98863199]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [26.15357041 35.37290026 32.13593276 43.37934436]\n",
      " [ 2.01645929  9.46511197 20.74991027  5.85208184]\n",
      " [ 1.35803848 30.45098359  8.23219173  4.63761884]\n",
      " [ 3.91711126 38.38553034  5.50661414  2.72217745]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.27226932 -0.5518115   3.93319997 -2.21027338]\n",
      " [ 1.99804948  3.11979613 10.03760369 -0.31770528]\n",
      " [21.86636748 -1.00222599  1.39558805  1.45297177]\n",
      " [-0.15249571 -1.96242421  0.68929249  0.08252013]\n",
      " [-0.12551501 -0.06515363  2.01870384 -0.81139608]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.        ]]\n",
      "Average Penalties in Last 100 Episodes: 0.29\n",
      "Episode: 600, Average Steps: 10.0\n",
      "Episode: 700\n",
      "Q-table:\n",
      "[[29.17170987 39.23945077 44.27100087 36.78533842]\n",
      " [19.84489019 27.37436546 46.38175909 24.45948712]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [28.13146944 39.72300782 37.95522193 46.84851913]\n",
      " [ 2.01645929  9.46511197 22.12668889  5.85208184]\n",
      " [ 1.35803848 33.90894028  8.23219173  4.63761884]\n",
      " [ 3.91711126 39.88096436  5.50661414  2.72217745]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.27226932 -0.5518115   3.93319997 -2.21027338]\n",
      " [ 1.99804948  3.11979613 10.03760369 -0.31770528]\n",
      " [21.86636748 -1.00222599  1.39558805  1.45297177]\n",
      " [-0.15249571 -1.96242421  0.68929249  0.08252013]\n",
      " [-0.12551501 -0.06515363  2.01870384 -0.81139608]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.        ]]\n",
      "Average Penalties in Last 100 Episodes: 0.05\n",
      "Episode: 700, Average Steps: 10.0\n",
      "Episode: 800\n",
      "Q-table:\n",
      "[[29.17170987 40.8264187  46.92531664 41.74577505]\n",
      " [19.84489019 27.37436546 49.15595132 24.45948712]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [30.59593056 40.55859133 41.01500447 49.49826475]\n",
      " [ 2.01645929  9.46511197 22.12668889  5.85208184]\n",
      " [ 1.35803848 33.90894028  8.23219173  4.63761884]\n",
      " [ 3.91711126 39.88096436  5.50661414  2.72217745]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.27226932 -0.5518115   3.93319997 -2.21027338]\n",
      " [ 1.99804948  3.11979613 10.03760369 -0.31770528]\n",
      " [21.86636748 -1.00222599  1.39558805  1.45297177]\n",
      " [-0.15249571 -1.96242421  0.68929249  0.08252013]\n",
      " [-0.12551501 -0.06515363  2.01870384 -0.81139608]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.        ]]\n",
      "Average Penalties in Last 100 Episodes: 0.05\n",
      "Episode: 800, Average Steps: 10.0\n",
      "Episode: 900\n",
      "Q-table:\n",
      "[[29.86512739 40.67390223 47.58131528 39.26699296]\n",
      " [19.84489019 27.37436546 49.93719026 24.45948712]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [33.25051423 42.31258557 41.01500447 50.17920522]\n",
      " [ 2.01645929  9.46511197 25.48004289  5.85208184]\n",
      " [ 1.35803848 37.43257001  8.23219173  4.63761884]\n",
      " [ 3.91711126 42.21281234  5.50661414  2.72217745]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.27226932 -0.5518115   3.93319997 -2.21027338]\n",
      " [ 1.99804948  3.11979613 10.03760369 -0.31770528]\n",
      " [21.86636748 -1.00222599  1.39558805  1.45297177]\n",
      " [-0.15249571 -1.96242421  0.68929249  0.08252013]\n",
      " [-0.12551501 -0.06515363  2.01870384 -0.81139608]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.        ]]\n",
      "Average Penalties in Last 100 Episodes: 0.11\n",
      "Episode: 900, Average Steps: 10.0\n",
      "Episode: 1000\n",
      "Q-table:\n",
      "[[29.16626439 40.67390223 49.11854347 41.48576772]\n",
      " [19.84489019 30.49336587 51.66496546 24.45948712]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [33.25051423 42.31258557 41.01500447 51.67166517]\n",
      " [ 2.01645929  9.46511197 27.14217769  5.85208184]\n",
      " [ 1.35803848 39.01322785  8.23219173  4.63761884]\n",
      " [ 3.91711126 42.21281234  5.50661414  2.72217745]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.27226932 -0.5518115   3.93319997 -2.21027338]\n",
      " [ 1.99804948  3.11979613 10.03760369 -0.31770528]\n",
      " [21.86636748 -1.00222599  1.39558805  1.45297177]\n",
      " [-0.15249571 -1.96242421  0.68929249  0.08252013]\n",
      " [-0.12551501 -0.06515363  2.01870384 -0.81139608]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.        ]]\n",
      "Average Penalties in Last 100 Episodes: 0.0\n",
      "Episode: 1000, Average Steps: 10.0\n"
     ]
    }
   ],
   "source": [
    "env_det = FrozenLakeEnv()\n",
    "\n",
    "epsilon = 1.0  # Initial exploration rate\n",
    "epsilon_min = 0.01  # Minimum exploration rate\n",
    "gamma = 0.95  # Discount factor\n",
    "alpha = 0.15  # Learning rate\n",
    "decay_rate = 0.995  # Epsilon decay rate per episode\n",
    "total_episodes = 1000 \n",
    "max_timestamp = 10\n",
    "qt = np.zeros((env_det.obs_space.n, env_det.action_space.n))\n",
    "\n",
    "rewards_epi = []\n",
    "epsilon_values = []\n",
    "steps_per_episode = []\n",
    "penalties_per_episode = []\n",
    "\n",
    "final_state = None\n",
    "for episode in range(total_episodes):\n",
    "    state, _ = env_det.reset()\n",
    "    state_index = env_det.obs_space_to_index(state)\n",
    "    total_rewards = 0\n",
    "    total_steps = 0  \n",
    "    action = env_det.action_space.sample() if np.random.uniform(0, 1) < epsilon else np.argmax(qt[state_index])\n",
    "\n",
    "    while True:\n",
    "        next_state, reward, terminated, truncated, _ = env_det.step(action)\n",
    "        total_steps += 1 \n",
    "        next_strt_idx = env_det.obs_space_to_index(next_state)\n",
    "        next_action = env_det.action_space.sample() if np.random.uniform(0, 1) < epsilon else np.argmax(qt[next_strt_idx])\n",
    "        qt[state_index, action] = qt[state_index, action] + alpha * (reward + gamma * qt[next_strt_idx, next_action] - qt[state_index, action])\n",
    "        state_index, action = next_strt_idx, next_action\n",
    "        total_rewards += reward\n",
    "        \n",
    "        if terminated or truncated:\n",
    "            break\n",
    "    \n",
    "    penalties_per_episode.append(env_det.get_penalty_count())  \n",
    "    \n",
    "    # Q-table for every 100 episodes\n",
    "    if (episode + 1) % 100 == 0:\n",
    "        print(f\"Episode: {episode + 1}\")\n",
    "        print(\"Q-table:\")\n",
    "        print(qt)\n",
    "        avg_penalty = np.mean(penalties_per_episode[-100:])\n",
    "        print(f\"Average Penalties in Last 100 Episodes: {avg_penalty}\")\n",
    "\n",
    "    epsilon = max(epsilon_min, epsilon * decay_rate)\n",
    "    epsilon_values.append(epsilon)\n",
    "    rewards_epi.append(total_rewards)\n",
    "    steps_per_episode.append(total_steps)\n",
    "\n",
    "    if (episode + 1) % 100 == 0:\n",
    "        average_steps = np.mean(steps_per_episode[-100:])\n",
    "        print(f\"Episode: {episode + 1}, Average Steps: {average_steps}\")\n",
    "\n",
    "    if episode == total_episodes - 1:\n",
    "        final_state = env_det.state\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 100\n",
      "Q-table 1:\n",
      "[[ 4.44203487  8.18633458 11.85416368  8.64172487]\n",
      " [ 3.46093516  9.34344823 12.0930265   6.09384082]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 1.20764956  2.26181567  5.08217587  9.07809385]\n",
      " [-0.89981292  2.7856339   5.11531748  2.19007779]\n",
      " [ 1.77219039  5.87305158  5.09193859  0.84162314]\n",
      " [ 0.60248264  8.51872763 -0.14901717  1.71720962]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.15       -0.128625    3.88382836 -0.7673582 ]\n",
      " [ 9.48435053  0.          0.2775     -0.01879452]\n",
      " [ 2.6021783   0.         -0.015       0.        ]\n",
      " [ 0.         -0.69493363  0.7068246   0.        ]\n",
      " [-0.015       0.06291875  3.25998486 -0.15      ]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.        ]]\n",
      "Q-table 2:\n",
      "[[ 3.9912651   8.14661117  9.98783913  8.92610903]\n",
      " [ 5.21168058  8.75845551 14.36755081  4.5665347 ]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 1.18601867  0.87867675  2.18465028 11.31758028]\n",
      " [-1.730722    1.47031361  4.64908018  0.32229384]\n",
      " [ 2.32121037  6.5567668   4.36935714  1.28520755]\n",
      " [ 4.74265278  8.27745111  0.          0.70413959]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 1.24060633  1.07199756  3.82046765 -0.36473917]\n",
      " [ 5.61270839  0.86559191  0.2775      0.40554839]\n",
      " [ 3.18921832  0.21453231 -0.015       1.09363834]\n",
      " [ 0.         -0.9         0.          0.        ]\n",
      " [-0.015      -0.15        3.58739299  0.        ]\n",
      " [ 0.          0.          0.          0.        ]\n",
      " [ 0.          0.          0.          0.        ]]\n",
      "Average Penalties in Last 100 Episodes: 0.63\n",
      "Episode: 100, Average Steps: 9.84\n",
      "Episode: 200\n",
      "Q-table 1:\n",
      "[[ 1.39789116e+01  1.96866109e+01  2.52138339e+01  2.01193086e+01]\n",
      " [ 9.28079170e+00  1.54739286e+01  2.70077807e+01  1.42619102e+01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 8.55149103e+00  1.71647164e+01  1.74316445e+01  2.77465107e+01]\n",
      " [ 1.45012419e+00  6.05798637e+00  1.12256552e+01  4.32342283e+00]\n",
      " [ 2.96736561e+00  1.87194734e+01  5.09193859e+00  1.83468459e+00]\n",
      " [ 2.19244396e+00  2.03509966e+01  2.71113559e+00  5.12336019e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 1.73453746e+00  1.90337244e+00  4.56321624e+00 -7.67358201e-01]\n",
      " [ 1.41238669e+01  1.43401394e+00  2.77500000e-01 -1.87945243e-02]\n",
      " [ 6.15051248e+00  0.00000000e+00 -1.50000000e-02  0.00000000e+00]\n",
      " [ 0.00000000e+00 -6.94933632e-01  1.47980765e+00  0.00000000e+00]\n",
      " [ 7.01256733e-01  7.37726371e-01  3.25998486e+00 -1.50000000e-01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]]\n",
      "Q-table 2:\n",
      "[[ 1.27170507e+01  1.94081929e+01  2.54761991e+01  1.84004083e+01]\n",
      " [ 1.20338211e+01  1.82021705e+01  2.65039382e+01  1.54584383e+01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 1.10971094e+01  1.73432149e+01  1.68476021e+01  2.78658386e+01]\n",
      " [ 1.50845999e+00  2.74039284e+00  1.12629796e+01  3.23813277e+00]\n",
      " [ 3.28745820e+00  1.49882581e+01  7.89607262e+00  2.03866494e+00]\n",
      " [ 6.78695050e+00  2.08777560e+01  0.00000000e+00  4.18041181e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 1.24060633e+00  3.34255858e+00  5.85435392e+00  7.31260487e-01]\n",
      " [ 5.61270839e+00  8.65591906e-01  7.56685408e-01  7.48161673e-01]\n",
      " [ 3.18921832e+00  2.14532309e-01 -1.50000000e-02  1.09363834e+00]\n",
      " [ 0.00000000e+00 -9.00000000e-01  6.14547842e-01  0.00000000e+00]\n",
      " [-1.50000000e-02  2.75945541e-01  7.35869343e+00 -4.92774940e-02]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]]\n",
      "Average Penalties in Last 100 Episodes: 0.52\n",
      "Episode: 200, Average Steps: 9.95\n",
      "Episode: 300\n",
      "Q-table 1:\n",
      "[[ 2.26445916e+01  3.14530691e+01  3.83381786e+01  3.15398629e+01]\n",
      " [ 1.47152601e+01  2.98881625e+01  3.79091435e+01  2.31309389e+01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 2.00264342e+01  2.70831729e+01  2.88605051e+01  3.78573171e+01]\n",
      " [ 5.50892226e+00  9.03139984e+00  1.31272139e+01  4.32342283e+00]\n",
      " [ 4.69293056e+00  2.21794945e+01  1.08643794e+01  1.83468459e+00]\n",
      " [ 4.91801938e+00  3.39018279e+01  8.85944990e+00  6.61130804e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 1.73453746e+00  1.90337244e+00  8.94323321e+00  3.05046158e+00]\n",
      " [ 2.45573697e+01  1.43401394e+00  2.77500000e-01 -1.87945243e-02]\n",
      " [ 1.10870317e+01  0.00000000e+00 -1.50000000e-02  0.00000000e+00]\n",
      " [ 0.00000000e+00 -6.94933632e-01  1.47980765e+00  0.00000000e+00]\n",
      " [ 7.01256733e-01  7.37726371e-01  3.25998486e+00 -1.50000000e-01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]]\n",
      "Q-table 2:\n",
      "[[ 2.05111990e+01  3.28298515e+01  3.53707149e+01  3.21698348e+01]\n",
      " [ 1.41227368e+01  2.19960490e+01  4.03850398e+01  2.32262421e+01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 2.24377565e+01  3.24428942e+01  3.02211916e+01  4.10078116e+01]\n",
      " [ 5.84522529e+00  2.74039284e+00  1.64316830e+01  3.23813277e+00]\n",
      " [ 3.28745820e+00  2.79853867e+01  9.76167874e+00  2.03866494e+00]\n",
      " [ 8.32831500e+00  3.36132167e+01  6.29755450e+00  6.18756684e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 1.24060633e+00  5.47539160e+00  7.13885187e+00  7.31260487e-01]\n",
      " [ 1.33963700e+01  8.65591906e-01  2.37308461e+00  1.13619574e+00]\n",
      " [ 1.37092719e+01  2.65324355e+00 -1.50000000e-02  1.09363834e+00]\n",
      " [ 0.00000000e+00 -9.00000000e-01  6.14547842e-01  0.00000000e+00]\n",
      " [-1.50000000e-02  2.75945541e-01  7.35869343e+00 -4.92774940e-02]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]]\n",
      "Average Penalties in Last 100 Episodes: 0.52\n",
      "Episode: 300, Average Steps: 9.75\n",
      "Episode: 400\n",
      "Q-table 1:\n",
      "[[ 2.23728574e+01  4.25559783e+01  4.32527684e+01  3.89853674e+01]\n",
      " [ 1.92874612e+01  3.37270276e+01  4.60032090e+01  2.57017572e+01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 2.73063333e+01  3.78448573e+01  3.88628590e+01  4.61972997e+01]\n",
      " [ 5.50892226e+00  9.03139984e+00  2.12334452e+01  6.12195437e+00]\n",
      " [ 5.15627737e+00  2.93174891e+01  1.08643794e+01  3.75099673e+00]\n",
      " [ 6.23929919e+00  4.01858516e+01  8.85944990e+00  6.61130804e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 1.73453746e+00  1.90337244e+00  1.02937762e+01  3.05046158e+00]\n",
      " [ 2.45573697e+01  1.43401394e+00  2.77500000e-01 -1.87945243e-02]\n",
      " [ 2.14007315e+01  0.00000000e+00 -1.50000000e-02  0.00000000e+00]\n",
      " [ 0.00000000e+00 -6.94933632e-01  1.47980765e+00  0.00000000e+00]\n",
      " [ 7.01256733e-01  7.37726371e-01  3.25998486e+00 -1.50000000e-01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]]\n",
      "Q-table 2:\n",
      "[[ 2.93540529e+01  4.16097620e+01  4.38043046e+01  3.87516158e+01]\n",
      " [ 1.73454108e+01  2.76210764e+01  4.57014336e+01  2.97070977e+01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 3.12790262e+01  3.95241788e+01  3.88617187e+01  4.57746771e+01]\n",
      " [ 5.84522529e+00  7.38694553e+00  1.72775085e+01  3.23813277e+00]\n",
      " [ 3.28745820e+00  3.33714387e+01  9.76167874e+00  2.03866494e+00]\n",
      " [ 8.32831500e+00  3.88570371e+01  6.29755450e+00  6.18756684e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 1.24060633e+00  5.47539160e+00  7.13885187e+00  7.31260487e-01]\n",
      " [ 2.21061125e+01  8.65591906e-01  2.37308461e+00  1.13619574e+00]\n",
      " [ 1.37092719e+01  2.65324355e+00 -1.50000000e-02  1.09363834e+00]\n",
      " [ 0.00000000e+00 -9.00000000e-01  6.14547842e-01  0.00000000e+00]\n",
      " [-1.50000000e-02  2.75945541e-01  7.35869343e+00 -4.92774940e-02]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]]\n",
      "Average Penalties in Last 100 Episodes: 0.28\n",
      "Episode: 400, Average Steps: 9.88\n",
      "Episode: 500\n",
      "Q-table 1:\n",
      "[[ 2.97299837e+01  4.32640943e+01  4.54112016e+01  4.01884464e+01]\n",
      " [ 2.80555288e+01  3.69113158e+01  4.83684979e+01  2.57017572e+01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 3.42281553e+01  4.26250449e+01  4.21720334e+01  4.84591105e+01]\n",
      " [ 5.50892226e+00  9.03139984e+00  2.94784092e+01  6.12195437e+00]\n",
      " [ 5.55012215e+00  3.66115669e+01  1.08643794e+01  3.75099673e+00]\n",
      " [ 6.23929919e+00  4.47266227e+01  8.85944990e+00  6.61130804e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 1.73453746e+00  1.90337244e+00  1.20498308e+01  3.05046158e+00]\n",
      " [ 2.45573697e+01  1.43401394e+00  2.77500000e-01 -1.87945243e-02]\n",
      " [ 2.14007315e+01  0.00000000e+00 -1.50000000e-02  0.00000000e+00]\n",
      " [ 0.00000000e+00 -6.94933632e-01  1.47980765e+00  0.00000000e+00]\n",
      " [ 7.01256733e-01  7.37726371e-01  3.25998486e+00 -1.50000000e-01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]]\n",
      "Q-table 2:\n",
      "[[ 3.08508623e+01  4.54806344e+01  4.56512756e+01  4.27120890e+01]\n",
      " [ 2.05383824e+01  3.58378638e+01  4.75922680e+01  3.39970292e+01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 3.44228431e+01  4.10364374e+01  4.18445104e+01  4.77250416e+01]\n",
      " [ 5.84522529e+00  7.38694553e+00  2.10814471e+01  3.23813277e+00]\n",
      " [ 3.28745820e+00  4.09111023e+01  9.76167874e+00  2.03866494e+00]\n",
      " [ 8.32831500e+00  4.12351486e+01  6.29755450e+00  6.18756684e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 1.24060633e+00  5.47539160e+00  7.13885187e+00  7.31260487e-01]\n",
      " [ 2.61573828e+01  8.65591906e-01  2.37308461e+00  1.13619574e+00]\n",
      " [ 1.37092719e+01  2.65324355e+00 -1.50000000e-02  1.09363834e+00]\n",
      " [ 0.00000000e+00 -9.00000000e-01  6.14547842e-01  0.00000000e+00]\n",
      " [-1.50000000e-02  2.75945541e-01  7.35869343e+00 -4.92774940e-02]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]]\n",
      "Average Penalties in Last 100 Episodes: 0.27\n",
      "Episode: 500, Average Steps: 9.97\n",
      "Episode: 600\n",
      "Q-table 1:\n",
      "[[ 3.66253754e+01  4.60894827e+01  4.81774366e+01  4.27744128e+01]\n",
      " [ 2.80555288e+01  3.97324613e+01  5.02251400e+01  2.57017572e+01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 3.51229233e+01  4.40886438e+01  4.45511246e+01  5.01028707e+01]\n",
      " [ 5.50892226e+00  9.03139984e+00  2.94784092e+01  6.12195437e+00]\n",
      " [ 5.55012215e+00  3.66115669e+01  1.08643794e+01  3.75099673e+00]\n",
      " [ 6.23929919e+00  4.72274625e+01  8.85944990e+00  6.61130804e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 1.73453746e+00  1.90337244e+00  1.20498308e+01  3.05046158e+00]\n",
      " [ 2.45573697e+01  1.43401394e+00  2.77500000e-01 -1.87945243e-02]\n",
      " [ 2.14007315e+01  0.00000000e+00 -1.50000000e-02  0.00000000e+00]\n",
      " [ 0.00000000e+00 -6.94933632e-01  1.47980765e+00  0.00000000e+00]\n",
      " [ 7.01256733e-01  7.37726371e-01  3.25998486e+00 -1.50000000e-01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]]\n",
      "Q-table 2:\n",
      "[[ 3.68575628e+01  4.69064466e+01  4.75294278e+01  4.37735633e+01]\n",
      " [ 2.05383824e+01  3.74987586e+01  5.04889643e+01  3.39970292e+01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 3.65729931e+01  4.18107866e+01  4.26152008e+01  5.06714372e+01]\n",
      " [ 5.84522529e+00  7.38694553e+00  2.10814471e+01  3.23813277e+00]\n",
      " [ 3.28745820e+00  4.09111023e+01  9.76167874e+00  2.03866494e+00]\n",
      " [ 8.32831500e+00  4.58137544e+01  6.29755450e+00  6.18756684e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 1.24060633e+00  5.47539160e+00  7.13885187e+00  7.31260487e-01]\n",
      " [ 2.61573828e+01  8.65591906e-01  2.37308461e+00  1.13619574e+00]\n",
      " [ 1.37092719e+01  2.65324355e+00 -1.50000000e-02  1.09363834e+00]\n",
      " [ 0.00000000e+00 -9.00000000e-01  6.14547842e-01  0.00000000e+00]\n",
      " [-1.50000000e-02  2.75945541e-01  7.35869343e+00 -4.92774940e-02]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]]\n",
      "Average Penalties in Last 100 Episodes: 0.1\n",
      "Episode: 600, Average Steps: 10.0\n",
      "Episode: 700\n",
      "Q-table 1:\n",
      "[[ 3.99112365e+01  4.79142613e+01  4.89639003e+01  4.39191551e+01]\n",
      " [ 2.99472539e+01  3.97324613e+01  5.14299087e+01  2.86071210e+01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 3.64898533e+01  4.46770306e+01  4.56438034e+01  5.14963289e+01]\n",
      " [ 5.50892226e+00  9.03139984e+00  3.10364799e+01  8.19276742e+00]\n",
      " [ 5.55012215e+00  3.95287694e+01  1.08643794e+01  3.75099673e+00]\n",
      " [ 6.23929919e+00  4.87118279e+01  8.85944990e+00  6.61130804e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 1.73453746e+00  1.90337244e+00  1.20498308e+01  3.05046158e+00]\n",
      " [ 2.45573697e+01  1.43401394e+00  2.77500000e-01 -1.87945243e-02]\n",
      " [ 2.14007315e+01  0.00000000e+00 -1.50000000e-02  0.00000000e+00]\n",
      " [ 0.00000000e+00 -6.94933632e-01  1.47980765e+00  0.00000000e+00]\n",
      " [ 7.01256733e-01  7.37726371e-01  3.25998486e+00 -1.50000000e-01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]]\n",
      "Q-table 2:\n",
      "[[ 3.90929524e+01  4.68227773e+01  4.90281279e+01  4.43472889e+01]\n",
      " [ 2.30488581e+01  3.89892761e+01  5.15350252e+01  3.39970292e+01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 3.72489956e+01  4.52933095e+01  4.50922471e+01  5.15578267e+01]\n",
      " [ 5.84522529e+00  7.38694553e+00  2.10814471e+01  3.23813277e+00]\n",
      " [ 3.28745820e+00  4.17547682e+01  9.76167874e+00  2.03866494e+00]\n",
      " [ 8.32831500e+00  4.76911695e+01  6.29755450e+00  6.18756684e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 1.24060633e+00  5.47539160e+00  7.13885187e+00  7.31260487e-01]\n",
      " [ 2.61573828e+01  8.65591906e-01  2.37308461e+00  1.13619574e+00]\n",
      " [ 1.37092719e+01  2.65324355e+00 -1.50000000e-02  1.09363834e+00]\n",
      " [ 0.00000000e+00 -9.00000000e-01  6.14547842e-01  0.00000000e+00]\n",
      " [-1.50000000e-02  2.75945541e-01  7.35869343e+00 -4.92774940e-02]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]]\n",
      "Average Penalties in Last 100 Episodes: 0.08\n",
      "Episode: 700, Average Steps: 10.0\n",
      "Episode: 800\n",
      "Q-table 1:\n",
      "[[ 4.17044501e+01  4.79830072e+01  4.61690734e+01  4.39191551e+01]\n",
      " [ 3.15552203e+01  4.11301428e+01  5.11292934e+01  2.86071210e+01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 3.71680177e+01  4.46770306e+01  4.66941219e+01  5.10261617e+01]\n",
      " [ 5.50892226e+00  9.03139984e+00  3.24810624e+01  8.19276742e+00]\n",
      " [ 5.55012215e+00  4.20110841e+01  1.08643794e+01  3.75099673e+00]\n",
      " [ 6.23929919e+00  4.91809771e+01  8.85944990e+00  6.61130804e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 1.73453746e+00  1.90337244e+00  1.20498308e+01  3.05046158e+00]\n",
      " [ 2.45573697e+01  1.43401394e+00  2.77500000e-01 -1.87945243e-02]\n",
      " [ 2.14007315e+01  0.00000000e+00 -1.50000000e-02  0.00000000e+00]\n",
      " [ 0.00000000e+00 -6.94933632e-01  1.47980765e+00  0.00000000e+00]\n",
      " [ 7.01256733e-01  7.37726371e-01  3.25998486e+00 -1.50000000e-01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]]\n",
      "Q-table 2:\n",
      "[[ 3.98255381e+01  4.69726127e+01  4.84325782e+01  4.63036235e+01]\n",
      " [ 2.30488581e+01  3.89892761e+01  5.23685857e+01  3.39970292e+01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 3.80402988e+01  4.58619543e+01  4.56897334e+01  5.21768635e+01]\n",
      " [ 5.84522529e+00  7.38694553e+00  2.10814471e+01  3.23813277e+00]\n",
      " [ 3.28745820e+00  4.17547682e+01  9.76167874e+00  2.03866494e+00]\n",
      " [ 8.32831500e+00  4.94193299e+01  6.29755450e+00  6.18756684e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 1.24060633e+00  5.47539160e+00  7.13885187e+00  7.31260487e-01]\n",
      " [ 2.61573828e+01  8.65591906e-01  2.37308461e+00  1.13619574e+00]\n",
      " [ 1.37092719e+01  2.65324355e+00 -1.50000000e-02  1.09363834e+00]\n",
      " [ 0.00000000e+00 -9.00000000e-01  6.14547842e-01  0.00000000e+00]\n",
      " [-1.50000000e-02  2.75945541e-01  7.35869343e+00 -4.92774940e-02]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]]\n",
      "Average Penalties in Last 100 Episodes: 0.09\n",
      "Episode: 800, Average Steps: 10.0\n",
      "Episode: 900\n",
      "Q-table 1:\n",
      "[[ 4.17044501e+01  4.79830072e+01  4.95636716e+01  4.50528430e+01]\n",
      " [ 3.15552203e+01  4.23323656e+01  5.21406707e+01  2.86071210e+01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 3.71680177e+01  4.60039692e+01  4.66941219e+01  5.21684117e+01]\n",
      " [ 5.50892226e+00  9.03139984e+00  3.37089575e+01  8.19276742e+00]\n",
      " [ 5.55012215e+00  4.29461657e+01  1.08643794e+01  3.75099673e+00]\n",
      " [ 6.23929919e+00  4.91809771e+01  8.85944990e+00  6.61130804e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 1.73453746e+00  1.90337244e+00  1.20498308e+01  3.05046158e+00]\n",
      " [ 2.45573697e+01  1.43401394e+00  2.77500000e-01 -1.87945243e-02]\n",
      " [ 2.14007315e+01  0.00000000e+00 -1.50000000e-02  0.00000000e+00]\n",
      " [ 0.00000000e+00 -6.94933632e-01  1.47980765e+00  0.00000000e+00]\n",
      " [ 7.01256733e-01  7.37726371e-01  3.25998486e+00 -1.50000000e-01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]]\n",
      "Q-table 2:\n",
      "[[ 3.86302588e+01  4.64908138e+01  4.95894967e+01  4.63036235e+01]\n",
      " [ 2.30488581e+01  3.89892761e+01  5.20788060e+01  3.39970292e+01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 3.80402988e+01  4.64008384e+01  4.64737471e+01  5.20272653e+01]\n",
      " [ 5.84522529e+00  7.38694553e+00  2.10814471e+01  3.23813277e+00]\n",
      " [ 3.28745820e+00  4.17547682e+01  9.76167874e+00  2.03866494e+00]\n",
      " [ 8.32831500e+00  4.94193299e+01  6.29755450e+00  6.18756684e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 1.24060633e+00  5.47539160e+00  7.13885187e+00  7.31260487e-01]\n",
      " [ 2.61573828e+01  8.65591906e-01  2.37308461e+00  1.13619574e+00]\n",
      " [ 1.37092719e+01  2.65324355e+00 -1.50000000e-02  1.09363834e+00]\n",
      " [ 0.00000000e+00 -9.00000000e-01  6.14547842e-01  0.00000000e+00]\n",
      " [-1.50000000e-02  2.75945541e-01  7.35869343e+00 -4.92774940e-02]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]]\n",
      "Average Penalties in Last 100 Episodes: 0.0\n",
      "Episode: 900, Average Steps: 10.0\n",
      "Episode: 1000\n",
      "Q-table 1:\n",
      "[[ 4.24760371e+01  4.79830072e+01  4.93927507e+01  4.56666637e+01]\n",
      " [ 3.15552203e+01  4.23323656e+01  5.20419674e+01  3.11722406e+01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 3.79469883e+01  4.60039692e+01  4.66941219e+01  5.20375974e+01]\n",
      " [ 5.50892226e+00  9.03139984e+00  3.37089575e+01  8.19276742e+00]\n",
      " [ 5.55012215e+00  4.29461657e+01  1.08643794e+01  3.75099673e+00]\n",
      " [ 6.23929919e+00  4.91809771e+01  8.85944990e+00  6.61130804e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 1.73453746e+00  1.90337244e+00  1.20498308e+01  3.05046158e+00]\n",
      " [ 2.45573697e+01  1.43401394e+00  2.77500000e-01 -1.87945243e-02]\n",
      " [ 2.14007315e+01  0.00000000e+00 -1.50000000e-02  0.00000000e+00]\n",
      " [ 0.00000000e+00 -6.94933632e-01  1.47980765e+00  0.00000000e+00]\n",
      " [ 7.01256733e-01  7.37726371e-01  3.25998486e+00 -1.50000000e-01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]]\n",
      "Q-table 2:\n",
      "[[ 3.86302588e+01  4.64908138e+01  4.95021316e+01  4.63036235e+01]\n",
      " [ 2.30488581e+01  3.89892761e+01  5.18368218e+01  3.39970292e+01]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 3.80402988e+01  4.67163197e+01  4.71785032e+01  5.19302899e+01]\n",
      " [ 5.84522529e+00  7.38694553e+00  2.10814471e+01  3.23813277e+00]\n",
      " [ 3.28745820e+00  4.17547682e+01  9.76167874e+00  2.03866494e+00]\n",
      " [ 8.32831500e+00  4.97547528e+01  6.29755450e+00  6.18756684e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 1.24060633e+00  5.47539160e+00  7.13885187e+00  7.31260487e-01]\n",
      " [ 2.61573828e+01  8.65591906e-01  2.37308461e+00  1.13619574e+00]\n",
      " [ 1.37092719e+01  2.65324355e+00 -1.50000000e-02  1.09363834e+00]\n",
      " [ 0.00000000e+00 -9.00000000e-01  6.14547842e-01  0.00000000e+00]\n",
      " [-1.50000000e-02  2.75945541e-01  7.35869343e+00 -4.92774940e-02]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]\n",
      " [ 0.00000000e+00  0.00000000e+00  0.00000000e+00  0.00000000e+00]]\n",
      "Average Penalties in Last 100 Episodes: 0.05\n",
      "Episode: 1000, Average Steps: 10.0\n"
     ]
    }
   ],
   "source": [
    "env_det = FrozenLakeEnv()\n",
    "epsilon = 1.0  \n",
    "epsilon_min = 0.01  \n",
    "gamma = 0.95 \n",
    "alpha = 0.15 \n",
    "decay_rate = 0.995  \n",
    "total_episodes = 1000\n",
    "max_timestamp = 10\n",
    "\n",
    "qt1 = np.zeros((env_det.obs_space.n, env_det.action_space.n))  # Q-table 1 initialization\n",
    "qt2 = np.zeros((env_det.obs_space.n, env_det.action_space.n))  # Q-table 2 initialization\n",
    "\n",
    "rewards_epi = []\n",
    "epsilon_values = []\n",
    "steps_per_episode = []\n",
    "penalties_per_episode = []\n",
    "\n",
    "final_state = None\n",
    "\n",
    "for episode in range(total_episodes):\n",
    "    state, _ = env_det.reset()\n",
    "    state_index = env_det.obs_space_to_index(state)\n",
    "    total_rewards = 0\n",
    "    total_steps = 0\n",
    "\n",
    "    while True:\n",
    "        total_steps += 1\n",
    "        action = env_det.action_space.sample() if np.random.uniform(0, 1) < epsilon else np.argmax((qt1[state_index] + qt2[state_index]) / 2)\n",
    "        next_state, reward, terminated, truncated, _ = env_det.step(action)\n",
    "        next_strt_idx = env_det.obs_space_to_index(next_state)\n",
    "        if np.random.uniform(0, 1) < 0.5:\n",
    "            qt1[state_index, action] += alpha * (reward + gamma * qt2[next_strt_idx, np.argmax(qt1[next_strt_idx])] - qt1[state_index, action])\n",
    "        else:\n",
    "            qt2[state_index, action] += alpha * (reward + gamma * qt1[next_strt_idx, np.argmax(qt2[next_strt_idx])] - qt2[state_index, action])\n",
    "        state_index = next_strt_idx\n",
    "        total_rewards += reward\n",
    "        if terminated or truncated or total_steps >= max_timestamp:\n",
    "            break\n",
    "\n",
    "    penalties_per_episode.append(env_det.get_penalty_count())\n",
    "    if (episode + 1) % 100 == 0:\n",
    "        print(f\"Episode: {episode + 1}\")\n",
    "        print(\"Q-table 1:\")\n",
    "        print(qt1)\n",
    "        print(\"Q-table 2:\")\n",
    "        print(qt2)\n",
    "        avg_penalty = np.mean(penalties_per_episode[-100:])\n",
    "        print(f\"Average Penalties in Last 100 Episodes: {avg_penalty}\")\n",
    "\n",
    "    epsilon = max(epsilon_min, epsilon * decay_rate)\n",
    "    epsilon_values.append(epsilon)\n",
    "    rewards_epi.append(total_rewards)\n",
    "    steps_per_episode.append(total_steps)\n",
    "\n",
    "    if (episode + 1) % 100 == 0:\n",
    "        average_steps = np.mean(steps_per_episode[-100:])\n",
    "        print(f\"Episode: {episode + 1}, Average Steps: {average_steps}\")\n",
    "\n",
    "    if episode == total_episodes - 1:\n",
    "        final_state = env_det.state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Q-Learning Training Loop\n",
    "q_rewards = []\n",
    "for episode in range(total_episodes):\n",
    "    state, _ = env_det.reset()\n",
    "    state_index = env_det.obs_space_to_index(state)\n",
    "    total_rewards = 0\n",
    "    total_steps = 0\n",
    "    action = env_det.action_space.sample() if np.random.uniform(0, 1) < epsilon else np.argmax(qt[state_index])\n",
    "    while True:\n",
    "        next_state, reward, terminated, truncated, _ = env_det.step(action)\n",
    "        total_steps += 1 \n",
    "        next_strt_idx = env_det.obs_space_to_index(next_state)\n",
    "        next_action = env_det.action_space.sample() if np.random.uniform(0, 1) < epsilon else np.argmax(qt[next_strt_idx])\n",
    "        qt[state_index, action] = qt[state_index, action] + alpha * (reward + gamma * qt[next_strt_idx, next_action] - qt[state_index, action])\n",
    "        state_index, action = next_strt_idx, next_action\n",
    "        total_rewards += reward\n",
    "        if terminated or truncated:\n",
    "            break\n",
    "    q_rewards.append(total_rewards)\n",
    "\n",
    "\n",
    "# Double Q-learning Training Loop\n",
    "double_q_rewards = []\n",
    "for episode in range(total_episodes):\n",
    "    state, _ = env_det.reset()\n",
    "    state_index = env_det.obs_space_to_index(state)\n",
    "    total_rewards = 0\n",
    "    total_steps = 0\n",
    "    while True:\n",
    "        total_steps += 1\n",
    "        action = env_det.action_space.sample() if np.random.uniform(0, 1) < epsilon else np.argmax((qt1[state_index] + qt2[state_index]) / 2)\n",
    "        next_state, reward, terminated, truncated, _ = env_det.step(action)\n",
    "        next_strt_idx = env_det.obs_space_to_index(next_state)\n",
    "        if np.random.uniform(0, 1) < 0.5:\n",
    "            qt1[state_index, action] += alpha * (reward + gamma * qt2[next_strt_idx, np.argmax(qt1[next_strt_idx])] - qt1[state_index, action])\n",
    "        else:\n",
    "            qt2[state_index, action] += alpha * (reward + gamma * qt1[next_strt_idx, np.argmax(qt2[next_strt_idx])] - qt2[state_index, action])\n",
    "        state_index = next_strt_idx\n",
    "        total_rewards += reward\n",
    "        if terminated or truncated or total_steps >= max_timestamp:\n",
    "            break\n",
    "    double_q_rewards.append(total_rewards)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlMAAAHHCAYAAACbXt0gAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8WgzjOAAAACXBIWXMAAA9hAAAPYQGoP6dpAACCpUlEQVR4nO3dd3wT5R8H8E+60r03tJS9ZxlWpqwyZQmiCAURBFGWypAfS6aoIMoSRFBAQRRQkSlL9gYBoewhUECgg9GZ5/dHScg1aZs045L28369AunNby6Xu+89z3PPKYQQAkRERERUIA5yB0BERERkz5hMEREREZmAyRQRERGRCZhMEREREZmAyRQRERGRCZhMEREREZmAyRQRERGRCZhMEREREZmAyRQRERGRCZhMFRIKhQITJkyQOwyTLVu2DBUqVICzszN8fX3lDsfm9e7dG1FRUXKHYVE7d+6EQqHAzp075Q7FIAqFAu+++26+0y1duhQKhQJXr161fFAFEBUVhd69e8sdRr6aNGmCJk2ayB0G5cGU85O590NLnSsLTTJ16dIlvP322yhVqhRcXV3h7e2N+vXrY/bs2Xj69Knc4ZEBzp07h969e6N06dJYtGgRFi5cmO88e/fuRadOnRASEgKlUomoqCgMGDAAN27cMHi9V69ehUKhwGeffWZK+CQzdXKifrm6uiI8PByxsbH48ssvkZKSIneIVnH9+nUMGDAAUVFRUCqVCA4ORqdOnbBv3z65QyvSmjRpotk3HRwc4O3tjfLly6Nnz57YunWrScv+4Ycf8MUXX5gn0EJgw4YNVi9ccLLq2izkjz/+QNeuXaFUKtGrVy9UqVIF6enp2LNnDz788EOcOXPGoBOzPXv69CmcnOz769y5cydUKhVmz56NMmXK5Dv9V199hSFDhqBUqVJ47733EBYWhrNnz+Kbb77BqlWrsHHjRrzwwgtWiFw+ixYtgkqlkjsMm/Lxxx+jZMmSyMjIQEJCAnbu3ImhQ4di5syZ+O2331CtWjW5Q7SYvXv3ok2bNgCAt956C5UqVUJCQgKWLl2KBg0aYO7cuRg4cKDMUZrXli1b5A7BYMWLF8e0adMAAI8fP8bFixexZs0aLF++HN26dcPy5cvh7Oxs9HJ/+OEHnD59GkOHDjVzxOZhyvkpPj4eDg7Glfts2LABc+fO1ZtQWepcad9nXwBXrlxB9+7dUaJECWzfvh1hYWGacYMGDcLFixfxxx9/yBih5ahUKqSnp8PV1RWurq5yh2Oyu3fvAoBB1Xt79+7F0KFD0aBBA2zatAnu7u6acQMHDkT9+vXRpUsXnDlzxm6qC4UQSE1NhZubm8HzFOTAW9i1bt0atWvX1vw9evRobN++He3atcPLL7+Ms2fPGrWN7cXDhw/xyiuvwM3NDXv37kXp0qU144YPH47Y2Fi89957qFmzps1eZGgf0wzl4uJiwYjMy8fHB2+88YZk2PTp0zF48GDMmzcPUVFR+OSTT2SKTqog30VuTFmGUqk0ef3aLHWutPtqvhkzZuDRo0dYvHixJJFSK1OmDIYMGaL5OzMzE5MmTULp0qU11UIfffQR0tLSJPNFRUWhXbt22LlzJ2rXrg03NzdUrVpV025jzZo1qFq1KlxdXREdHY3jx49L5u/duzc8PT1x+fJlxMbGwsPDA+Hh4fj4448hhJBM+9lnn+HFF19EQEAA3NzcEB0djZ9//lnns6jbYqxYsQKVK1eGUqnEpk2bNOO0s/CUlBQMHTpUUtTfokULHDt2TLLM1atXIzo6Gm5ubggMDMQbb7yBmzdv6v0sN2/eRMeOHeHp6YmgoCB88MEHyMrKyuWbkZo3b54m5vDwcAwaNAiJiYmS7T1+/HgAQFBQUL712pMmTYJCocB3330nSaQAoHTp0pgxYwZu3bpl1hLJtLQ0jB8/HmXKlIFSqURERARGjBihs+8sWbIETZs2RXBwMJRKJSpVqoT58+frLE+9j23evFmzj3399deaNkI//fQTpkyZguLFi8PV1RXNmjXDxYsXJcvI2WZKu8py4cKFmv28Tp06OHz4sE4Mq1evRqVKleDq6ooqVapg7dq1BrfD+vXXX9G2bVuEh4dDqVSidOnSmDRpks4+0aRJE1SpUgX//PMPXnrpJbi7u6NYsWKYMWOGzjL//fdfdOzYER4eHggODsawYcN0tm9BNG3aFGPHjsW1a9ewfPlyybjt27ejYcOG8PDwgK+vLzp06ICzZ89Kpsltm0yYMAEKhULvOlesWIHy5ctrjhF//fWXQbFu3LhRE4+Xlxfatm2LM2fO5Dvf119/jYSEBHz66aeSRAoA3Nzc8N133wHILrkrqMTERAwdOhQRERFQKpUoU6YMPvnkE53SUVOPaeoq271792L48OEICgqCh4cHOnXqhHv37kmWkbPNlDG/HwCYO3cuSpUqBTc3N9StWxe7d++2ajssR0dHfPnll6hUqRLmzJmDpKQkyfjly5drjtH+/v7o3r27pBlDkyZN8Mcff+DatWuaakTtfdXQ41Z+38WePXswePBgBAUFwdfXF2+//TbS09ORmJiIXr16wc/PD35+fhgxYoTOOS7n8Vz9u7l48SJ69+4NX19f+Pj4oE+fPnjy5Ilk3pxtpjIyMjBx4kSULVsWrq6uCAgIQIMGDTRVpb1798bcuXM161W/cosFAG7evIm+fftqjmUlS5bEwIEDkZ6enveXp03YuWLFiolSpUoZPH1cXJwAIF555RUxd+5c0atXLwFAdOzYUTJdiRIlRPny5UVYWJiYMGGCmDVrlihWrJjw9PQUy5cvF5GRkWL69Oli+vTpwsfHR5QpU0ZkZWVJ1uPq6irKli0revbsKebMmSPatWsnAIixY8dK1lW8eHHxzjvviDlz5oiZM2eKunXrCgBi/fr1kukAiIoVK4qgoCAxceJEMXfuXHH8+HHNuPHjx2umff3114WLi4sYPny4+Oabb8Qnn3wi2rdvL5YvX66ZZsmSJQKAqFOnjpg1a5YYNWqUcHNzE1FRUeLhw4c6n6Vy5crizTffFPPnzxddunQRAMS8efPy3ebjx48XAETz5s3FV199Jd59913h6Ogo6tSpI9LT04UQQqxdu1Z06tRJABDz588Xy5YtEydPntS7vMePHwsnJyfRpEmTXNeZmpoqlEqlaNCgQb7xXblyRQAQn376aa7TZGVliZYtWwp3d3cxdOhQ8fXXX4t3331XODk5iQ4dOkimrVOnjujdu7eYNWuW+Oqrr0TLli0FADFnzhzJdCVKlBBlypQRfn5+YtSoUWLBggVix44dYseOHQKAqFmzpoiOjhazZs0SEyZMEO7u7qJu3bqSZcTFxYkSJUrofJaaNWuKMmXKiE8++UTMmDFDBAYGiuLFi2u2txBCrF+/XigUClGtWjUxc+ZMMXbsWOHn5yeqVKkiWWZuOnbsKLp16yY+/fRTMX/+fNG1a1cBQHzwwQeS6Ro3bizCw8NFRESEGDJkiJg3b55o2rSpACA2bNigme7JkyeiXLlywtXVVYwYMUJ88cUXIjo6WlSrVk0AEDt27MgzHvX+fPjwYb3jb9y4ofntq23dulU4OTmJcuXKiRkzZoiJEyeKwMBA4efnJ65cuZLrdlZT79vaAIgqVaqIwMBA8fHHH4tPPvlElChRQri5uYlTp07pxKu9nu+//14oFArRqlUr8dVXX4lPPvlEREVFCV9fX8l0+rz44ovC1dVVpKam5jpN48aNhbOzs3j69GmeyxIie/+Mi4vT/P348WNRrVo1ERAQID766COxYMEC0atXL6FQKMSQIUMk85p6TFNvm5o1a4qmTZuKr776Srz//vvC0dFRdOvWTeczNW7cWPO3Mb+fefPmCQCiYcOG4ssvvxTDhw8X/v7+onTp0pJlmkPjxo1F5cqVcx0/adIknW00efJkoVAoxKuvvirmzZun2T+1j9FbtmwRNWrUEIGBgWLZsmVi2bJlYu3atUII445b+X0XNWrUEK1atRJz584VPXv2FADEiBEjRIMGDcTrr78u5s2bpznHfffddzrL1j4/qX83NWvWFJ07dxbz5s0Tb731lmaZ2nLuhx999JFQKBSiX79+YtGiReLzzz8Xr732mpg+fboQQoh9+/aJFi1aCACa7bFs2bJcY7l586YIDw/XbKMFCxaIsWPHiooVK0rOg/mx62QqKSlJANDZKXJz4sQJAUC89dZbkuEffPCBACC2b9+uGVaiRAkBQOzbt08zbPPmzQKAcHNzE9euXdMM//rrr3UO9uqk7b333tMMU6lUom3btsLFxUXcu3dPM/zJkyeSeNLT00WVKlVE06ZNJcMBCAcHB3HmzBmdz5ZzB/Hx8RGDBg3KdVukp6eL4OBgUaVKFcmBdf369QKAGDdunM5n+fjjjyXLUB+s8nL37l3h4uIiWrZsKUk258yZIwCIb7/9VjNM/QPT3jb6qL/HnAfwnKpVqyb8/f3znEYIw5KpZcuWCQcHB7F7927J8AULFggAYu/evZphOb9PIYSIjY3VSfrV+9imTZskw9Ung4oVK4q0tDTN8NmzZwsAkhNybslUQECAePDggWb4r7/+KgCI33//XTOsatWqonjx4iIlJUUzbOfOnQKAQcmUvs/59ttvC3d3d8kJvXHjxgKA+P777zXD0tLSRGhoqOjSpYtm2BdffCEAiJ9++kkz7PHjx6JMmTJmSaaEyP5d1KxZU/N3jRo1RHBwsLh//75m2MmTJ4WDg4Po1auXZpixyRQAceTIEc2wa9euCVdXV9GpUyedeNVJUkpKivD19RX9+vWTLC8hIUH4+PjoDM/J19dXVK9ePc9pBg8eLACIv//+O8/phNA9iU2aNEl4eHiI8+fPS6YbNWqUcHR0FNevX9cMM/WYpt42zZs3FyqVSjN82LBhwtHRUSQmJmqG5ZZM5ff7SUtLEwEBAaJOnToiIyNDM93SpUsFAKsnU2vXrhUAxOzZs4UQQly9elU4OjqKKVOmSKY7deqUcHJykgxv27at3v3TmONWft9FbGys5LuIiYkRCoVCDBgwQDMsMzNTFC9eXGfb5ZZMvfnmm5LpOnXqJAICAiTDcu6H1atXF23bttX5rNoGDRqk87vMLZZevXoJBwcHvccN7c+bH7uu5ktOTgYAeHl5GTT9hg0bAGS3H9D2/vvvA4BO26pKlSohJiZG83e9evUAZFcZREZG6gy/fPmyzjq1b5FWF6Omp6fjzz//1AzXbr/x8OFDJCUloWHDhjpVcgDQuHFjVKpUKZ9Pmt3u6ODBg7h165be8UeOHMHdu3fxzjvvSOqQ27ZtiwoVKuhtZzZgwADJ3w0bNtT7mbX9+eefSE9Px9ChQyWNCPv16wdvb+8CtWdT35WV3/fu5eVltju4Vq9ejYoVK6JChQr477//NK+mTZsCAHbs2KGZVvv7TEpKwn///YfGjRvj8uXLOkX4JUuWRGxsrN519unTR9IepGHDhgD072c5vfrqq/Dz88t13lu3buHUqVPo1asXPD09NdM1btwYVatWzXf5gPRzpqSk4L///kPDhg3x5MkTnDt3TjKtp6enpK2Ii4sL6tatK/ksGzZsQFhYGF555RXNMHd3d/Tv39+geAzh6emp2Sdu376NEydOoHfv3vD399dMU61aNbRo0UJzvCiImJgYREdHa/6OjIxEhw4dsHnz5lyrxrdu3YrExES89tprkn3M0dER9erVk+xj+qSkpBj0m1BPa6zVq1ejYcOG8PPzk8TXvHlzZGVlSaoxzXVM69+/v6SKpmHDhsjKysK1a9fyjTe/38+RI0dw//599OvXT9IguUePHpLfjrWof4fq72bNmjVQqVTo1q2bZHuHhoaibNmy+e4PgHHHLSDv76Jv376S76JevXoQQqBv376aYY6Ojqhdu7ZBxyhA/znl/v37mnO7Pr6+vjhz5gwuXLhg0DryolKpsG7dOrRv317SzlIttyp8fey6Abq3tzcAww8M165dg4ODg86dYqGhofD19dX5gWonTEB240EAiIiI0Dv84cOHkuEODg4oVaqUZFi5cuUAQNK3zPr16zF58mScOHFCUo+t74ssWbJkrp9P24wZMxAXF4eIiAhER0ejTZs26NWrlyYe9WctX768zrwVKlTAnj17JMNcXV0RFBQkGebn56fzmXPKbT0uLi4oVaqUQQfFnAw9IaSkpCA4OFjz97179yQnMk9PT0kikZcLFy7g7NmzOttATd14HshuHD9+/Hjs379fp/4/KSlJs78AeX+fOfc/9QE+v21uyLzq7a7vrskyZcroPenldObMGfzvf//D9u3bdQ5+OZPG4sWL6+zPfn5++PvvvzV/X7t2DWXKlNGZTt8+WlCPHj3S7BN5/QYqVqyIzZs34/Hjx/Dw8DB6PWXLltUZVq5cOTx58gT37t1DaGioznj1yUF9ostJfbzLjSEXD+rx6m2QlJQk6TrGxcVFkljmjO/vv/826DdgrmOaHL8BJycng9oMPnjwQNKmxs3NTfLbNtajR48APD++XbhwAUIIvfsSYNjNJ8YctwDjvou8zoeGfD/6lqn9HeW2v3/88cfo0KEDypUrhypVqqBVq1bo2bNnge7SvXfvHpKTk1GlShWj583J7pOp8PBwnD592qj5DM02HR0djRoucjS6M8Tu3bvx8ssvo1GjRpg3bx7CwsLg7OyMJUuW4IcfftCZ3tC7kLp164aGDRti7dq12LJlCz799FN88sknWLNmDVq3bm10nLl9ZjmULVsWTk5OkhNxTmlpaYiPj0fdunU1w+rUqSNJ3saPH29wXyQqlQpVq1bFzJkz9Y5XH1AuXbqEZs2aoUKFCpg5cyYiIiLg4uKCDRs2YNasWToNdfP6Pk3Zz8y5j+qTmJiIxo0bw9vbGx9//DFKly4NV1dXHDt2DCNHjtT5nJaOxxD//vsvkpKSDOp2I6fcjhmG3oBhCPU2W7Zsmd5kK7/buStVqoRjx44hLS0t1zug/v77b7i4uKBYsWIAgCFDhmgapgPZJRO5dY6qUqnQokULjBgxQu949YWiOY9ptvwb6Ny5M3bt2qX5Oy4uDkuXLi3w8tTnMfX+qVKpoFAosHHjRr2fxZALQUOPW2oF+S70DTd0GxfkO2rUqBEuXbqEX3/9FVu2bME333yDWbNmYcGCBXjrrbcMWq8l2HUyBQDt2rXDwoULsX//fkmVnD4lSpSASqXChQsXULFiRc3wO3fuIDExESVKlDBrbCqVCpcvX9YcZADg/PnzAKC58vnll1/g6uqKzZs3Sw6AS5YsMXn9YWFheOedd/DOO+/g7t27qFWrFqZMmYLWrVtrPmt8fLzOlXB8fLzZtoX2erRL6dLT03HlyhU0b97c6GW6u7ujWbNm+PPPP3Ht2jW9sf70009IS0tD165dNcNWrFghuQrPWWqYl9KlS+PkyZNo1qxZnsn477//jrS0NPz222+Sqy5DiuStSb3N9N3dpG9YTjt37sT9+/exZs0aNGrUSDP8ypUrJsV0+vRpCCEk2zg+Pr7Ay9S2bNkyANBUq2rvmzmdO3cOgYGBmlIpPz8/yd2narmVrOqrgjh//jzc3d1zLSVQ34EXHBxcoN9F+/btsW/fPqxevVrn9nsguzR89+7d6NChg+akOWLECMm0eVVvlS5dGo8ePco3Nkse08xJ+zfw0ksvaYZnZmbi6tWr+ZZ0fP7555ISmPDw8ALHkpWVhR9++AHu7u5o0KABgOztLYRAyZIlJecQfXI7Jhl63LI3/v7+6NOnD/r06YNHjx6hUaNGmDBhgiaZMvSzBgUFwdvb2+gCGX3sus0UkH0w8PDwwFtvvYU7d+7ojL906RJmz54NAJrO7HL2FKvO2tu2bWv2+ObMmaN5L4TAnDlz4OzsjGbNmgHIzswVCoXkCvfq1atYt25dgdeZlZWlU80SHByM8PBwTZF77dq1ERwcjAULFkiK4Tdu3IizZ8+abVs0b94cLi4u+PLLLyVXG4sXL0ZSUlKB1/O///0PQgj07t1bp4f7K1euYMSIEYiIiEDPnj01w+vXr4/mzZtrXsYkU926dcPNmzexaNEinXFPnz7F48ePATy/0tL+rElJSTZ3IgkPD0eVKlXw/fffa6oXAGDXrl04depUvvPr+5zp6emYN29egWNq06YNbt26JbmF/smTJ2bp3mL79u2YNGkSSpYsiR49egDIvtioUaMGvvvuO0midPr0aWzZskVzvACyT0pJSUmS0tDbt29j7dq1ete3f/9+SVXpjRs38Ouvv6Jly5a5Xo3HxsbC29sbU6dORUZGhs74nF0C5PT2228jNDQUH374oU6bldTUVPTp0wcKhUJSslSpUiXJb0K7nVdO3bp1w/79+7F582adcYmJicjMzARgmWOaJdSuXRsBAQFYtGiRJnYg+6LLkGqq6OhoybYzpC2rPllZWRg8eDDOnj2LwYMHa6q3OnfuDEdHR0ycOFGnpEYIgfv372v+9vDw0DnmA4Yft+yJ9ucGskvoypQpIzmPqS+C9F0AaXNwcEDHjh3x+++/48iRIzrjjSnFtPuSqdKlS+OHH37Aq6++iooVK0p6QFdfpan7qKhevTri4uKwcOFCTTXFoUOH8N1336Fjx46SqxNzcHV1xaZNmxAXF4d69eph48aN+OOPP/DRRx9prk7btm2LmTNnolWrVnj99ddx9+5dzJ07F2XKlMmzGisvKSkpKF68OF555RVUr14dnp6e+PPPP3H48GF8/vnnALLr2z/55BP06dMHjRs3xmuvvYY7d+5g9uzZiIqKwrBhw8yyDYKCgjB69GhMnDgRrVq1wssvv4z4+HjMmzcPderU0XsFbYgGDRpg1qxZGDp0KKpVq4bevXsjLCwM586dw6JFi+Dg4IB169YZ1WHntm3bkJqaqjO8Y8eO6NmzJ3766ScMGDAAO3bsQP369ZGVlYVz587hp59+0vQV1bJlS7i4uKB9+/Z4++238ejRIyxatAjBwcG4fft2gT6rpUydOhUdOnRA/fr10adPHzx8+BBz5sxBlSpVJAmWPi+++CL8/PwQFxeHwYMHQ6FQYNmyZSZVofTr1w9z5sxBr169cPToUYSFhWHZsmU6/YjlZ+PGjTh37hwyMzNx584dbN++HVu3bkWJEiXw22+/SW64+PTTT9G6dWvExMSgb9++ePr0Kb766iv4+PhIqoC7d++OkSNHolOnThg8eDCePHmC+fPno1y5cnrbl1WpUgWxsbEYPHgwlEqlJsmcOHFirnF7e3tj/vz56NmzJ2rVqoXu3bsjKCgI169fxx9//IH69etLLs5y8vPzw88//4w2bdqgVq1aOj2gX758GXPmzNHcMGOsDz/8EL/99hvatWuH3r17Izo6Go8fP8apU6fw888/4+rVqwgMDLTIMc0SXFxcMGHCBLz33nto2rQpunXrhqtXr2Lp0qUoXbq0RUpykpKSNP2cPXnyRNMD+qVLl9C9e3dMmjRJM23p0qUxefJkjB49GlevXkXHjh3h5eWFK1euYO3atejfvz8++OADANmJ3apVqzB8+HDUqVMHnp6eaN++vcHHLXtSqVIlNGnSBNHR0fD398eRI0fw888/S272Ul8UDB48GLGxsXB0dET37t31Lm/q1KnYsmULGjdujP79+6NixYq4ffs2Vq9ejT179hh+DjH4vj8bd/78edGvXz8RFRUlXFxchJeXl6hfv7746quvJLdpZ2RkiIkTJ4qSJUsKZ2dnERERIUaPHq3TN0uJEiX03n4JQKfLAX231sfFxQkPDw9x6dIlTT8fISEhYvz48ZIuAoQQYvHixaJs2bJCqVSKChUqiCVLluR6y3Vu3R1A63bPtLQ08eGHH4rq1asLLy8v4eHhIapXr663T6hVq1aJmjVrCqVSKfz9/UWPHj3Ev//+K5lG/Vly0hdjbubMmSMqVKggnJ2dRUhIiBg4cKBOHx6Gdo2gbffu3aJDhw4iMDBQKBQKAUAEBweL27dvG7wM9feX20vdR0l6err45JNPROXKlYVSqRR+fn4iOjpaTJw4USQlJWmW99tvv4lq1aoJV1dXERUVJT755BPx7bff6vQplNs+pr61e/Xq1XrjXLJkiWZYbl0j6OvmQXsfUVu5cqWoUKGCUCqVokqVKuK3334TXbp0ERUqVMh3u+3du1e88MILws3NTYSHh4sRI0Zoug/R7sYgt1vC9XU3cO3aNfHyyy8Ld3d3ERgYKIYMGSI2bdpkVNcI6peLi4sIDQ0VLVq0ELNnzxbJycl65/vzzz9F/fr1hZubm/D29hbt27cX//zzj850W7ZsEVWqVBEuLi6ifPnyYvny5Xn+TpcvX675XdesWVMnfn39TAmR/f3HxsYKHx8f4erqKkqXLi169+4t6WohL1evXhX9+/cXkZGRwsnJSbM9/vzzT4PmV8t5S7oQ2d03jB49WpQpU0a4uLiIwMBA8eKLL4rPPvtM0oeZqce03Lq5UP82cu5f+rpGMOT3I4QQX375pShRooRQKpWibt26Yu/evSI6Olq0atXKgK1kOHUXIeqXp6enKFu2rHjjjTfEli1bcp3vl19+EQ0aNBAeHh7Cw8NDVKhQQQwaNEjEx8drpnn06JF4/fXXha+vr07XJoYet4z9LnI7Xus7X+Q89uQ2r77fRM79cPLkyaJu3brC19dXuLm5iQoVKogpU6ZI9r/MzEzx3nvviaCgIM15IbdYhMg+7vTq1UsEBQUJpVIpSpUqJQYNGiTpWiM/imcLJzPr3bs3fv7553yv8Mm8Jk2ahHHjxmHMmDGYPHmy3OHYpRo1aiAoKMjkh6+Sbdi2bRvatGmDBg0aYOPGjXb1+BU5qFQqBAUFoXPnznqrx4j0sfs2U0Taxo4diwEDBmDKlCmF/uHWpsrIyJC0FQGyG5afPHnSao/SIMtr1qwZvvvuO+zYsQN9+vSx6h2Uti41NVVne3z//fd48OABfwNkFJZMWQhLpsjWXb16Fc2bN8cbb7yB8PBwnDt3DgsWLICPjw9Onz6NgIAAuUMksqidO3di2LBh6Nq1KwICAnDs2DEsXrwYFStWxNGjR1mKRwaz+wboRFQwfn5+iI6OxjfffIN79+7Bw8MDbdu2xfTp05lIUZEQFRWFiIgIfPnll3jw4AH8/f3Rq1cvTJ8+nYkUGYUlU0REREQmYJspIiIiIhMwmSIiIiIyAdtM5aBSqXDr1i14eXkVqu73iYiICjMhBFJSUhAeHg4HB+uWFTGZyuHWrVs6D38kIiIi+3Djxg0UL17cqutkMpWDl5cXgOwvQ/2MJCIiIrJtycnJiIiI0JzHrYnJVA7qqj1vb28mU0RERHZGjiY6bIBOREREZAImU0REREQmYDJFREREZAK2mSqArKwsZGRkyB0GkcGcnZ3h6OgodxhERIUSkykjCCGQkJCAxMREuUMhMpqvry9CQ0PZfxoRkZkxmTKCOpEKDg6Gu7s7T0pkF4QQePLkCe7evQsACAsLkzkiIqLChcmUgbKysjSJVEBAgNzhEBnFzc0NAHD37l0EBwezyo+IyIzYAN1A6jZS7u7uMkdCVDDqfZft/YiIzIvJlJFYtUf2ivsuEZFlMJkiIiIiMoHdJlPTp0+HQqHA0KFDNcNSU1MxaNAgBAQEwNPTE126dMGdO3fkC5IsrkmTJpJ9gIiIyNrsMpk6fPgwvv76a1SrVk0yfNiwYfj999+xevVq7Nq1C7du3ULnzp1litL23LhxA2+++SbCw8Ph4uKCEiVKYMiQIbh//36e802YMAE1atSwTpBGWrNmDSZNmiR3GEREVITZXTL16NEj9OjRA4sWLYKfn59meFJSEhYvXoyZM2eiadOmiI6OxpIlS7Bv3z4cOHBAxohtw+XLl1G7dm1cuHABP/74Iy5evIgFCxZg27ZtiImJwYMHD+QOUcLQRtL+/v6yPCGcKKen6Vlyh0CFlLX2LZVKIDWD+3FB2F0yNWjQILRt2xbNmzeXDD969CgyMjIkwytUqIDIyEjs378/1+WlpaUhOTlZ8iqMBg0aBBcXF2zZsgWNGzdGZGQkWrdujT///BM3b97EmDFjCrzsGzduoFu3bvD19YW/vz86dOiAq1evasYfPnwYLVq0QGBgIHx8fNC4cWMcO3ZMsgyFQoH58+fj5ZdfhoeHB6ZMmaIpEVu2bBmioqLg4+OD7t27IyUlRTNfzmq+qKgoTJ06FW+++Sa8vLwQGRmJhQsXSta1b98+1KhRA66urqhduzbWrVsHhUKBEydOFHgbUNE24bczqDhuE45ffyh3KFTIrDt+ExXHbcKyA9csvq4e3xxEhbGbcP9RmsXXVdjYVTK1cuVKHDt2DNOmTdMZl5CQABcXF/j6+kqGh4SEICEhIddlTps2DT4+PppXRESEwfEIIfAkPVOWlxDC4DgfPHiAzZs345133tH0N6QWGhqKHj16YNWqVUYtUy0jIwOxsbHw8vLC7t27sXfvXnh6eqJVq1ZIT08HAKSkpCAuLg579uzBgQMHULZsWbRp00aSFAHZ1YmdOnXCqVOn8OabbwIALl26hHXr1mH9+vVYv349du3ahenTp+cZ0+eff47atWvj+PHjeOeddzBw4EDEx8cDAJKTk9G+fXtUrVoVx44dw6RJkzBy5EijPzeRtqX7rgIAZm49L28gVOgMXXUCADB23WmLr2v/5ewmH1v+YVtjY9lNp503btzAkCFDsHXrVri6upptuaNHj8bw4cM1fycnJxucUD3NyEKlcZvNFosx/vk4Fu4uhn19Fy5cgBACFStW1Du+YsWKePjwIe7du4fg4GCj4li1ahVUKhW++eYbza33S5Ysga+vL3bu3ImWLVuiadOmknkWLlwIX19f7Nq1C+3atdMMf/3119GnTx/JtCqVCkuXLtVU5fXs2RPbtm3DlClTco2pTZs2eOeddwAAI0eOxKxZs7Bjxw6UL18eP/zwAxQKBRYtWgRXV1dUqlQJN2/eRL9+/Yz63ERERGp2UzJ19OhR3L17F7Vq1YKTkxOcnJywa9cufPnll3ByckJISAjS09N1npt3584dhIaG5rpcpVIJb29vyauwyq/kKTU1FZ6enprX1KlT813myZMncfHiRXh5eWnm8/f3R2pqKi5dugQg+zvo168fypYtCx8fH3h7e+PRo0e4fv26ZFm1a9fWWX5UVJSkTVRYWJjmsSi50b4xQaFQIDQ0VDNPfHw8qlWrJknI69atm+/nJCIiyo3dlEw1a9YMp06dkgzr06cPKlSogJEjRyIiIgLOzs7Ytm0bunTpAiD7xHn9+nXExMRYJCY3Z0f883GsRZZtyLoNVaZMGSgUCpw9exadOnXSGX/27FkEBQUhPDxc0m7I398/32U/evQI0dHRWLFihc64oKAgAEBcXBzu37+P2bNno0SJElAqlYiJidFUA6p5eHjoLMPZ2Vnyt0KhgEqlyjOmgsxDRERUUHaTTHl5eaFKlSqSYR4eHggICNAM79u3L4YPHw5/f394e3vjvffeQ0xMDF544QWLxKRQKAyuapNTQEAAWrRogXnz5mHYsGGSdlMJCQlYsWIFBg0aBCcnJ5QpU8aoZdeqVQurVq1CcHBwrqV6e/fuxbx589CmTRsA2VW2//33X8E/kAnKly+P5cuXIy0tDUqlEkB2A3kiIqKCsptqPkPMmjUL7dq1Q5cuXdCoUSOEhoZizZo1codlE+bMmYO0tDTExsbir7/+wo0bN7Bp0ya0aNEC5cqVw7hx4/Kc/+nTpzhx4oTkdenSJfTo0QOBgYHo0KEDdu/ejStXrmDnzp0YPHgw/v33XwBA2bJlsWzZMpw9exYHDx5Ejx49dBrCW8vrr78OlUqF/v374+zZs9i8eTM+++wzAHzcChERFYxdJ1M7d+7EF198ofnb1dUVc+fOxYMHD/D48WOsWbMmz/ZSRUnZsmVx+PBhlCpVCt26dUOJEiXQunVrlCtXTnMHXl7Onz+PmjVrSl5vv/023N3d8ddffyEyMhKdO3dGxYoV0bdvX6SmpmpKqhYvXoyHDx+iVq1a6NmzJwYPHmx0Q3dz8fb2xu+//44TJ06gRo0aGDNmjCaRNOeNDUREVHQoREHuhy/EkpOT4ePjg6SkJEm1VWpqKq5cuYKSJUsWmpPu+PHjMXPmTGzdutViVaH2YMWKFejTpw+SkpJkKzGzhsK4D9uKqFF/AAAalg3Esr71ZI6GChP1vgUAV6e3tcq6pnWuitfqRlp0XZaQ2/nbGmy/wQ9ZzMSJExEVFYUDBw6gbt26cHCw64JKg33//fcoVaoUihUrhpMnT2LkyJHo1q1boU6kiIjIcphMFXE5+3UqChISEjBu3DgkJCQgLCwMXbt2zbPfKiIiorwwmaIiZ8SIERgxYoTcYRARUSFRNOp1iIiIiCyEyRQRERGRCZhMEREREZmAyRQRERGRCZhMEREREZmAyRQRERGRCZhMkdk1adIEQ4cOzXOaqKgoyaOA5KBQKLBu3TpZY1AzZJsREZFtYjJVBPTu3RsKhQIKhQLOzs4ICQlBixYt8O2330KlUskdnknWr1+Pxo0bw8vLC+7u7qhTpw6WLl0qd1hGW7NmDSZNmiR3GEREVABMpoqIVq1a4fbt27h69So2btyIl156CUOGDEG7du2QmZkpd3gF8tVXX6FDhw6oX78+Dh48iL///hvdu3fHgAED8MEHH8gdHgAgIyPDoOn8/f3h5eVl4WiIiMgSmEwVEUqlEqGhoShWrBhq1aqFjz76CL/++is2btwoKcm5fv06OnToAE9PT3h7e6Nbt264c+eOZnzv3r3RsWNHybKHDh2KJk2aSIZlZmbi3XffhY+PDwIDAzF27Fjk9UztxMREvPXWWwgKCoK3tzeaNm2KkydP5jr9jRs38P7772Po0KGYOnUqKlWqhDJlyuD999/Hp59+is8//xwHDx40ahvduHED3bp1g6+vL/z9/dGhQwdcvXpVM/7w4cNo0aIFAgMD4ePjg8aNG+PYsWOSZSgUCsyfPx8vv/wyPDw8MGXKFEyYMAE1atTAsmXLEBUVBR8fH3Tv3h0pKSma+XJW80VFRWHq1Kl488034eXlhcjISCxcuFCyrn379qFGjRpwdXVF7dq1sW7dOigUCpw4ccKoz01ERKZhMmUKIYD0x/K88khMDNW0aVNUr14da9asAQCoVCp06NABDx48wK5du7B161ZcvnwZr776qtHL/u677+Dk5IRDhw5h9uzZmDlzJr755ptcp+/atSvu3r2LjRs34ujRo6hVqxaaNWuGBw8e6J3+559/RkZGht4SqLfffhuenp748ccfDY43IyMDsbGx8PLywu7du7F37154enqiVatWSE9PBwCkpKQgLi4Oe/bswYEDB1C2bFm0adNGkhQBwIQJE9CpUyecOnUKb775JgDg0qVLWLduHdavX4/169dj165dmD59ep4xff7556hduzaOHz+Od955BwMHDkR8fDyA7Kejt2/fHlWrVsWxY8cwadIkjBw50uDPS0RE5sNn85ki4wkwNVyedX90C3DxMHkxFSpUwN9//w0A2LZtG06dOoUrV64gIiICAPD999+jcuXKOHz4MOrUqWPwciMiIjBr1iwoFAqUL18ep06dwqxZs9CvXz+daffs2YNDhw7h7t27UCqVAIDPPvsM69atw88//4z+/fvrzHP+/Hn4+PggLCxMZ5yLiwtKlSqF8+fPGxzvqlWroFKp8M0330ChUAAAlixZAl9fX+zcuRMtW7ZE06ZNJfMsXLgQvr6+2LVrF9q1a6cZ/vrrr+s8QFqlUmHp0qWaqryePXti27ZteT5guU2bNnjnnXcAACNHjsSsWbOwY8cOlC9fHj/88AMUCgUWLVoEV1dXVKpUCTdv3tS7fYmIyLJYMlXECSE0ycPZs2cRERGhSaQAoFKlSvD19cXZs2eNWu4LL7ygWS4AxMTE4MKFC8jKytKZ9uTJk3j06BECAgLg6empeV25cgWXLl0q4CfLTqoAYOrUqZLlXr9+XW8MFy9ehJeXl2Y6f39/pKamamK4c+cO+vXrh7Jly8LHxwfe3t549OiRzvJq166ts/yoqChJm6iwsDDcvXs3z/irVaumea9QKBAaGqqZJz4+HtWqVYOrq6tmmrp16+a3SYiIyAJYMmUKZ/fsEiK51m0GZ8+eRcmSJQ2e3sHBQaftk6GNrHPz6NEjhIWFYefOnTrjfH199c5TtmxZJCUl4datWwgPl5YOpqen49KlS4iNjQUADBgwAN26ddOMzzm9Oobo6GisWLFCZ1xQUBAAIC4uDvfv38fs2bNRokQJKJVKxMTEaKoB1Tw8dEsMnZ2dJX8rFIp876QsyDxERGR9TKZMoVCYpapNLtu3b8epU6cwbNgwAEDFihVx48YN3LhxQ1M69c8//yAxMRGVKlUCkJ1YnD59WrKcEydO6Jz4czb+VrcxcnR01ImjVq1aSEhIgJOTE6KiogyK/ZVXXsHIkSPx+eef4/PPP5eMW7BgAZ48eYJevXoByL5Tzt/fP8/l1apVC6tWrUJwcDC8vb31TrN3717MmzcPbdq0AZDdYP2///4zKF5zK1++PJYvX460tDRN1ejhw4dliYWIqKhjNV8RkZaWhoSEBNy8eRPHjh3D1KlT0aFDB7Rr106TdDRv3hxVq1ZFjx49cOzYMRw6dAi9evVC48aNNVVXTZs2xZEjR/D999/jwoULGD9+vE5yBWTfFTh8+HDEx8fjxx9/xFdffYUhQ4boja158+aIiYlBx44dsWXLFly9ehX79u3DmDFjcOTIEb3zREZGYsaMGfjiiy8wZswYnDt3DpcuXcLMmTMxYsQITJ48GVWqVDF4+/To0QOBgYHo0KEDdu/ejStXrmDnzp0YPHgw/v33XwDZpWHLli3D2bNncfDgQfTo0QNubm4Gr8OcXn/9dahUKvTv3x9nz57F5s2b8dlnnwGApHqViIgsj8lUEbFp0yaEhYUhKioKrVq1wo4dO/Dll1/i119/1ZQWKRQK/Prrr/Dz80OjRo3QvHlzlCpVCqtWrdIsJzY2FmPHjsWIESNQp04dpKSkaJIxbb169cLTp09Rt25dDBo0CEOGDNHbkFy93g0bNqBRo0bo06cPypUrh+7du+PatWsICQnJ9TMNGzYMa9aswe7du1G7dm1N1whLly7FRx99ZNT2cXd3x19//YXIyEh07twZFStWRN++fZGamqopqVq8eDEePnyIWrVqoWfPnhg8eDCCg4ONWo+5eHt74/fff8eJEydQo0YNjBkzBuPGjQMASTsqIiKyPIXIq/OfIig5ORk+Pj5ISkqSVPekpqbiypUrKFmyJE9WNurBgwdo1qwZvL29sXHjRri7m6ddmb1YsWIF+vTpg6SkJL0lZtyHLSdq1B8AgIZlA7Gsbz2Zo6HCRL1vAcDV6W2tsq5pnavitbqRFl2XJeR2/rYGlkxRoeHv748///wTzZo1w/79++UOx+K+//577NmzB1euXMG6deswcuRIdOvWTbaqRyKioooN0KlQCQgI0FR3FXYJCQkYN24cEhISEBYWhq5du+bZbxUREVkGkykiOzVixAiMGDFC7jCIqBBgix/TsJqPiIioiGMuZRomU0Zi9k72ivsuEeWGRwfTMJkykLpTyidPnsgcCVHBqPfdnB2sEhHxYss0bDNlIEdHR/j6+mqejebu7s7OEckuCCHw5MkT3L17F76+vnp7oSeioo2plGmYTBkhNDQUAPJ9QC2RLfL19dXsw0RE2lgwZRomU0ZQKBQICwtDcHCwyQ/3JbImZ2dnlkgRUa4Ey6ZMwmSqABwdHXliIiKiQoMlU6ZhA3QiIiIiEzCZIiIiKuJYMmUaJlNERERFHNtMmYbJFBERkbWpsuSOQMLkkikhAJXKLLHYIyZTRERE1rTpI2BGSSDpptyRaJhcLvVde2DeC0BWpjnCsTt2k0zNnz8f1apVg7e3N7y9vRETE4ONGzdqxqempmLQoEEICAiAp6cnunTpgjt37sgYMRERkR4H5gKpScC+L+WORMPkHtCv7gb+iwfunTVPQHbGbpKp4sWLY/r06Th69CiOHDmCpk2bokOHDjhz5gwAYNiwYfj999+xevVq7Nq1C7du3ULnzp1ljpqIiMj2mZRKsfW6/fQz1b59e8nfU6ZMwfz583HgwAEUL14cixcvxg8//ICmTZsCAJYsWYKKFSviwIEDeOGFF+QImYiIyC6YlA8xmbKfkiltWVlZWLlyJR4/foyYmBgcPXoUGRkZaN68uWaaChUqIDIyEvv3789zWWlpaUhOTpa8iIiIihSTkqmi2/Bcza6SqVOnTsHT0xNKpRIDBgzA2rVrUalSJSQkJMDFxQW+vr6S6UNCQpCQkJDnMqdNmwYfHx/NKyIiwoKfgIiIyPaY1DUCkyn7SqbKly+PEydO4ODBgxg4cCDi4uLwzz//mLTM0aNHIykpSfO6ceOGmaIlIiIqArSTqSJa5Wc3baYAwMXFBWXKlAEAREdH4/Dhw5g9ezZeffVVpKenIzExUVI6defOHYSGhua5TKVSCaVSacmwiYiIbJppbaZYMmVXJVM5qVQqpKWlITo6Gs7Ozti2bZtmXHx8PK5fv46YmBgZIyQiIrJ9ppUnac2tUJgYiX2ym5Kp0aNHo3Xr1oiMjERKSgp++OEH7Ny5E5s3b4aPjw/69u2L4cOHw9/fH97e3njvvfcQExPDO/mIiIjyYVI/UyyZsp9k6u7du+jVqxdu374NHx8fVKtWDZs3b0aLFi0AALNmzYKDgwO6dOmCtLQ0xMbGYt68eTJHTUREZPu0Uymj8yomU/aTTC1evDjP8a6urpg7dy7mzp1rpYiIiIgKB+0Eyug7+5hM2XebKSIiIjKddgJlfMmU0P++CGEyRUREVNQJvW8NnJclU0ymiIiIijhJAmVs6VIRLY3SxmSKiIioiBPmKpkqoqVUTKaIiIiKONPaTDGZYjJFRERUxEnbkJtwN18RrfJjMkVERFTEiVzeGzYzS6aYTBERERVx2qVRrOYzHpMpIiKiIo4N0E3DZIqIiIg0jH9On0mVhIUCkykiIqIizqR245JiLZZMERERURHErhFMw2SKiIioiDPbg46ZTBEREVFRJGn1xJIpozGZIiIiKuIkXSMYPTOTKSZTRERERZxpJVOm9KtQODCZIiIiKuLYZso0TKaIiIisxWafXce7+UzBZIqIiMhaCmOywWSKyRQREZHV2GiywU47TcNkiojIBMY/eoOKNBtNNqQN0NlmylhMpoiITMBcioxiozuMpHDJ+L4RcnlfdDCZIiIyQdE8dVCB2WjJjeRxMkbPzJIpJlNERCZgNR8ZxUaTDZNKpphMMZkiIjIFUykyio0mG+brZ6po/iKYTBERmaCInjuooGw1mWI/UyZhMkVElpWZBlw/CGRlyh2JRRh9FU9Fm5HJhiOyEK2IhzNy/H7+uwAk3jBfWKY8EYbJFJMpskEqFfDzm8DOT+SOxP4JAfw6CNg6Tr4Y1r0DfNsS2DFFvhgsiCVTZBQjd5jRTj/gF+VETHX65vnAJw+AObWBL6qYObhn2DWC0ZhMke25thc4/Quwc6rckdi//y4Ax5cDe2fLd9Y//XP2//u+lGf9RDbFuJbebzltBAB0dfrr+cDEa0Ytw6CoTCqZMqlfhUKByRTZnsxUuSMoPLLSnr9XZckXRyFWRM8dVFCSkpsC7jzaO53KPNXnprWZYg/oTKaIigpVhtwRFEpsM0VGMXeVmLmSKbPdzcdkisj28LLffLKYTFkCd1EyijkSD4Xi+Xsz/a6lj5MxdmYmU0ymyLYxATAfM13BkhRzKTKKzZZMsQd0UzCZItvGqinTaLeTYmJqEewBnYxijsRDpTUfS6ZsApMpsm1MAEyjfdXKxNQimEqRUcxx55v2b9kW2kzxV8Bkimwcq6ZMo739mJhaBAumyCjmePSKRS6STOgbgSVTTKbIBjEBMJ8s81/BUg5MpsgY5kg8tH/XZnqyAHtAN43dJFPTpk1DnTp14OXlheDgYHTs2BHx8fGSaVJTUzFo0CAEBATA09MTXbp0wZ07d2SK2AQ3jwEbRgBPH8odiTwkCQCTKZNobz8mphbBrhHIKObok8kCJVPSNlPsGsFYdpNM7dq1C4MGDcKBAwewdetWZGRkoGXLlnj8+LFmmmHDhuH333/H6tWrsWvXLty6dQudO3eWMeoCWvQScOhrYNNHckciDwu0ByiytK9aZd+WivwnsUOs5iOjmL1kykzJlClNudhpJ5zkDsBQmzZtkvy9dOlSBAcH4+jRo2jUqBGSkpKwePFi/PDDD2jatCkAYMmSJahYsSIOHDiAF154QY6wTXPvrNwRyEM7ASikD8e1GpUtlfIVzqyjcH4qshiz3M1n/oskdo1gGrspmcopKSkJAODv7w8AOHr0KDIyMtC8eXPNNBUqVEBkZCT2798vS4wmK6qXvDaVANg5C7StICl2jUDGEbm8N4Il7ubTfm9SyVTR/D3YTcmUNpVKhaFDh6J+/fqoUiX7qdkJCQlwcXGBr6+vZNqQkBAkJCTkuqy0tDSkpT1/fllycrJFYi6YorlTsgG6GbFrBIsror9SKiizVPOZ/xjJx8mYxi5LpgYNGoTTp09j5cqVJi9r2rRp8PHx0bwiIiLMECGZhHegmY9NJaaFs80UkVHMUs1nY6X3TKbsL5l69913sX79euzYsQPFixfXDA8NDUV6ejoSExMl09+5cwehoaG5Lm/06NFISkrSvG7cuGGp0MlQNpUA2DmbSkwLZxkOazjIKLbaNYLW79O0HtCL5o/AbpIpIQTeffddrF27Ftu3b0fJkiUl46Ojo+Hs7Ixt27ZphsXHx+P69euIiYnJdblKpRLe3t6Sl80oojslu0YwI3aNYHGSk1AhTRjJjMzeaaeZLpJM2XVZMmU/baYGDRqEH374Ab/++iu8vLw07aB8fHzg5uYGHx8f9O3bF8OHD4e/vz+8vb3x3nvvISYmxj7v5CvKmACYTxbbTFkcS6bIGGa/m4/9TNkCu0mm5s+fDwBo0qSJZPiSJUvQu3dvAMCsWbPg4OCALl26IC0tDbGxsZg3b56VIyWT2VTfSHbOphLTwtlmyqS7oKjoMUefTJbuZ8romZlM2U0yZUim7Orqirlz52Lu3LlWiIgshp12mo+kyjRLvjgKMdMeEEtFjq0+6NiUNlPm6O7BztlNm6miqWjulGyAbkY21TVC4dyfTTsJUZFTKLtGYA/oTKbI9tjUHWh2Tu7EtAhkFyZVj1DRY6s9oGu/N+luvqKZTNlNNR8Z7ml6FtxcHPOdLkslkKlSQZl6H8hMNdPaFYCrD5CaqHdsaoYKrs755PBPteZNSQASr+tO4+YPPH0AuPnpPhBa6Q2kPwaEVrWWwhFQegKpSQZ9igJxcgUUDkDGE8utw1ja2/LxPf3b0hLU30vOg35qkuY7SM1UwcVBAQcH7bZUevafnN+n+rvPjbN79tkg82neMbp4Alnp2S8A6ZkqOCgAJw+/7PHqfUXpBTg4ZX8ePfubQ3IqiuEeACAwM8P4baxwyI4lrYAdBju5Zf+v/ryOyux4Mx7nPk9eXDyBzGcdGTspgfRH+c/j7J59EnVSAmkp2d9hfg9qd3LN/l0qFNnHn/ySAjc/IO1R7jHpOxYURH77l5tf9vZx8QQe3zV++Sm3n7/PeGLc/qKeNvnW82GP7+a/DAO2jcuj+5r92CfdBUj00ZnmqcIDboo03e/qyX2tiRJ14hFCIC1TSI/9PhHZ330hoRB8FoJEcnIyfHx8kJSUJF83CROe7cQhVYGBe4ya9Z9byWjz5W70iimBjztUyXPadl/tRt37v2IcFhU0UiIiIuP97x7g5GLWRcp5/mbJlE0zPs+dve08AOD7/dfyTaZO30zGG04XsvcCByfAwbkgQT4nVECW+tE8iuwrTy1PM56XFLk551Nypr7KVl916xunTT1dVtrzYmZHl+wr35zz6FumqVQZz6/WFI7Z65Zbzu1kic+dk2QfeLZOVWaO9loKZDoqkZGZ/T1p9gWRpSkl0uw/Ob9PzXjo/zza34ODc/Z+rU9W+vOSLgdnqBwckZahggsy4Kgw4Hfn5Ar13YkqCKRlZMfooFBA6WRE6wntz6xwyC5VMobk8zplx6Te1gXZD7W3t1p+cWnHkJPWdpKQfNfP5PZ95dyndGIS0pJ1U/bz/I4TeR17CrIeA+ZNzciEqyIDacIJSmdn3W2X3zIMPPZlCYH0Z79JRwcFXByl+3FWRurz34a+7yqPz6Q+9utbbmHBZKqIysx6dvBXJ2xN/wc0GGbaQu9fAr6qlf2+ZEMg7nfJ6Iqj/tC8vzqxbcHXk/EUiN8IhFYF7pwGyrfJLvoHgJU9gHPrs9+/vgoo3TT7/YPLwL14oFwryxQt758HbB6d/b7h+0DTMeZfh7E+rwikPKsO+PAS4BFo+XUm3wZmVsh+7xMJDDuV/f7GIWBxi+z3pZrg96pzMGzVSQBa+8LNo8CiZ99XRF2g7xbgx9eA+A3Zw177EQivBVzZBZRrDThLk3WN6wcBB0egeO3c49w6Htj7Rfb7jvNwPbwtmny2E4ucP0MLx2PZw3uuy64eWdhYd/5RNzRX1TcfPEHDGTsAANEl/PDLwBfz2EA5XN4FfP9y9vtyrYHXfjB8XgDYMwv4c0L2+0YjAN9IYN2A7L8bDAOajTVueb8NBo59Jx1W/XWgYx53SP/1KbB9cvb7Mi2Ai1ufjxt9E3DM5TRz6wSwdgBw72z2368uB8q30p1O+7iiL6bMNGBy8PNx/8v9Waz5SksBLmwFyjQHXPWUbMx94Xm8QHZ15ijLVp1X0HfcvHksO4EsYcC+lpIA3DiYfZx0zP2Cede5O3hz6REAQPc6EZjepZpk/O9jW6Oj477sP15dBpRvbfBnUB/7O9cshpmv1jB4PnvCZKqISlcnU+orDYUZrhactK5enXI50ZmDsxtQpXP2+8Cy0nEOWiVeCq33/qWyX5aivV6H/NurWZ/MbRPy2D5CCChyS3C190uFA+DuD1TulPe6IuvlH4/2/umkRKYq+3eg0r4nx8Ep9+9Sa7j0TncjS5Ml26UAv8Gc28fU32BB9l3t9VTulH3iVrf/yi2RAoDwGtkXO+rkxCmX0i99+4al2toovZ4fW/TJraTT2orVyn8aNa9QoFKHfCfLr8eGLO3fhsIWj3HyKpzlbYWFBZuzqaslFHhWpG+WZErroCpXNZf2wc6aBz5bT6as1dBTez2SduXa28cJCq2RGVlC3wx6lmfGw1WOpENdvaHSjsHBMfd1ag2XPk7GSKbur5JkSqGTJBpNXwz57Tra68yttDDXeQ1I/vR9B5JhVrxQ0FlV4WlAnV/XCFnCDMe4wrO5dDCZKqLS1HXjZk2mtA6Mct2loZApqVHkUiJG2bRP0jm2T1pmXp2J6kmwzEGSdLhqYtC5+s41mXoei0l9MJq635i7ZKogMZiyTmet9jW5lkzlk0wVojvC5JRf1wiS34YtXjDKjMlUEZX6rEGgps2UWZIpKzRwzk9u1XzWXK8tHmisdsLJJfnJUZ2l0jpaqxN7aSmUumQqR7JgLs7SZCo1Q10yleOEYcA+ZFLfzw4mnqBybh/t5MTYUqJcY8hn33EyICHKdV6t6Z1zOX7YUslUToUokdOuota3H6vMUc1XiPsOYDJVRKlPYM+TKTMkAHm1j7AWuZIauaoXDSbzQT9HNZ+6Wg3QSqb0zmehZCpHdZj+kikHg9YpTCmaMmc1n4OjTG2mTFinIdWS+o5NLJkyu/xKpjJztickCSZTNs34NN7QxrDqk8fzNlNmPiDJ1X0Zq/n0k6XNVC4lUwpHSQKVqukyI782U+as5pOWiKRl5lYylf86TSqZkuw35miAbmKbKb2JS34lU9rrdDPut1/QNlOS37acyVThSeTyazOl89swkPrOcQCFaXPpYDJViKnvUNJHt2SqkOwKrObLhW3dzafdTkp9M4ReViuZyo4hSxjYZkqLSW2mTN1v8kym5CiZMjKB0+7brsAN0Mk88n4uUkHv5suz5LkQ4R5pywpQuqN9EZmakXvDXvU4szZAzy0Qa+LdfPrJ3WYqRzVfaoZ2Nd+z/dSabaZylIiofw86jWwNWqd2WxMZq/nMkkzpiyGffUfSiNzVuH1Ne9qCJlNyVvNZed2WfGBJfs+YzETBjnGSZIptpsheaBdG5XVF8LxrhEJWMsVqvlzIXTKlfTefg6RkKtXQkilzfgbteLRKpnQa2RqwD5l2N5+JjXrzSqYK0j2JqXfzFaTRu77laMuvn6ki1GbKkq0npG2mpCvKUokCV/NpX9SnZxXeUqpCcgYtrIz/5RjasNfi1XxytZliA3T9bKnN1LNHt6g9T6z0lWpZqJ8p7f3TyQ1pJpRM5ddwN09m72eqAO2kJPEUoM2UdtLm5GrcRtCeNrffq01X81m5ZMqSy86jZCo9U1XgBuhpht5sYuds8KhPOWWpBBKfpOsd5+iggNLJEU/Ss5+N9Tjt+TOy7iSnwjWX54Q9eJz9rCuHZ9V8j9JVSHuUpndaYwQ8+z8tS4VHWsvzUEp3tfsFWJenqxMepeb9ZHn3TEBd6fDwaRZUBqzHkOXmxzlNBfXDJ1LSVUg3w7Y0la8QmoL5+4/TgXTjf+7GbhvFkzT4P3ufpRJIfLYdFE+zNMNTMwWSMp8/q++/R+m4/ygNjk/T4ftsWEaWCsmP0uCRJaAu60hMzUSWmbarc2qG5vu6nwokPsmORzuZevg0EyIjUxO3Nu39Vz0vAGSqVEbt2w5Ps+D37H26UCAll3ndXZyQlpmFrGdFz+rvRZmuguezaR6lC6SlKTS/waSn6ch8lAYPpROepGcZVEXkmiHgkWNYakYWHufxmRyfZGi+t/tpCvhBaLZifttCmZapiT/XadOzNJ9J7WmGwBOt6bXHF+TYknP75sZHJSQnTZUAHj5KM/kYolAo4O7iKDl+63P/URocHSyTwKWkPt+P0zKk+3HS0wxJyZShx1YAuJv8/LmJj1IzNcv193DJ/ckHdojJlI3LUgm0nv0Xzt95ZPS8nefty3cadcnU2N/OYu26P41eR05Xn535tpx7gPdO5b686Mmmr0ufD52uY9CzvbrNV/txG+ctsp6cGjv8je+eXaCP+fUsfltrmc9njP3KNIQ9O1a9+MlOpMHyvdL7IRnHn+0D1x88wUvPvmdvPMbfz4b/cvRfrM78VzPPB6uzn9FXUXENG58VrBy5lojXJv+J6U630f3Z9/nG4kM4Lf4zS5wxDmfw47PNET1lm2a49gmj3Zz9SIMLjuipucpt/z1/55FR+3YwHuLQs+Uv3nsNn+wybr/p5HAWs559jkkb4rHq952a3+Cb3x3FUWHccaOv4yWMzfH4tl+O3cKYQ7nHVVpxE9uefW8vzNiDfUoHBD3b7/LbFh0dzuELl7yndUMqzub4Dr7Zdx2f//V8+qta4y11bAGAP1xSUFnr+vT+k3TUseD6cqo7dVv+E5nBpjMJ2HRG+ozD97QeUNxpwQFcFdeMXu7+y/c138/5ya3h4lR4kilbKSslfUR2iVRBEilDqdtMqcxUXD0l43XcFAH4JLO7WZZnLO1GkplW3L2115UFW2wzJa+sAn4XQmu/FGb8Pg+ryuOUKgrrsqQPis2SfI8OZvtd5EZ7fZkF2G+041O//z3rBZxUlcIJUaYA8Rgfw2URhkOq8tiaVQsZcMLb6cNxR/ji3fT38p13k6oOzquKYUVms1ynUen53i39vZCuLMmxlce4nBTCkrcH2KHk5GT4+PggKSkJ3t56nhpuDRN8sv8PLI97cbtRZ0p2Jn91elvJZDvi76LPksMAgLbVwjD3dSMefqn2XXvgyl9Al8VA1VdMCjs3XRfsw+GrDwEAfwxugMrhPkYv415KGnadv4fWVUJ1qgwldk4Hdk7Lfv/BRcAzKM/lJqdmYPPpBDSvGAI/DxNKbq78lb0tAaDb9wY9WNTiPq8IpNzKfv+/u0a1p3nwOB21Jm0FABT3c8OekU0Nm/HJA2BGyez3/qWAwcez32c8BaaEZr+v/SbQbpbuvAmngQX1s99HNQR6rwd+HwIcXZo9bMAeILSqwZ+hQLaOA/bOzn7/fnx2eyD159E2Ick869PaXl9mdsTd2h9gckfpZ1SpBP44dRsVQr0Q7O2K6hO3AADCfFyxv91D4Je+2RN2mAfU7CGZNy0zC3/8fRt1ovwR4e+efzyHFgEbPpAMSqzUE77d5hTwA5pBZhowOVg6rPEo4KXRz/+eoHVMMfK7UakENpy+jXIhXigX4pX3xAsaAAmnnv/tEYwnQ85iw6kENCobiGBv4xvgX7ybguYz/wIA1Ir0xZp36hu9DKvY8wXw5/js90NPA74Rsoajj5znb1bz2Th1rquvntxRq77ZsaB1z+pc2oJ11w5ay3ZyKFjpQpCXEq9EFzdyxflfPXm7OqNrbTMcFArZ3XwF3p9yXb3WNjHm+s3ajw3J+T1auqFzjuXr+304OCjQvno4AOi2qcnnQdBKJ0d0rmXE70bPMmRv1mLhBugODgq0qxZe4PndXZyMPzZpcdT6zp0cbbiyyNa7f5GZDX9zBDzv6kBfm0MnB4Xe90YR6n6mLPfjcHLUSvqsucfxbr7njDwjOjqa+QxqyHeh905AC93NZwiD+5kyZR3SfcU5n+3ulHN8zsfJmDmeZysxfbmmyO9xMnIyQ6apfezO7/uXla0f42RmI3sk6Sc0D4TVd9eDg9aP0MHkZMpyu4J2yZSDNS9z2QO6FrlLpgrYyaKlOu00dN0WT6ak+0p+JRPOOUuuzL199HaNYPpiTZJfP1N2zlnrOy9oyb1VmNonWiFnUHo5fPhwgxc4c+bMAgdDOQihuVVXX66kXfVX8Go+yydTjpISNCseLKzaaaeMJ31DGLl/FPhrym092sNzrebT92w+GberNUqmcpyU8ith1rloklRLmiHB0HuSlLtkSs/6beaCxfRt42iOGgZrs+WkTyYGJVPHjx+X/H3s2DFkZmaifPnyAIDz58/D0dER0dHR5o+wiBOaaj49baa0k6mCFg+r1I/ysNyPw0lSgmax1eiy6uNkbL0I3Lj9w2aukOXs6drAHtBNkmNfMXq7m71kSnfftclCIFu8YCkg7ao9u+l3ySaPcfIyaIvs2LFD837mzJnw8vLCd999Bz+/7O7mHj58iD59+qBhw4aWibIIU1fz6U2mzNIA3brVfJbqcE4vVvM9Z2zJVIG/JgNmNKT0ytLP5jOEVdpM5WiAbuxFkbm3j97vxgZP8LaSTJmjzZQtNzrPDav5dBj9LX7++eeYNm2aJpECAD8/P0yePBmff/65WYMjoZVM6Y6VlEzZcJsps8RZEFatUrTxkikjD/o2c4Us5wNtrXE3Xw5GV/NYIZlSMJmyKLup2tNmi8c4mRm9RyYnJ+PevXs6w+/du4eUlBSzBEXPae7m09c1goMCJRQJ+MNlNKomFrBnXHUyZcHEw8Ecbbtsnc13jWAlJn2/+tpM6Xten5VYo2QqB6NLKazRKNgWf7NyJVM6zf3Mezef3bDF0neZGb1HdurUCX369MGaNWvw77//4t9//8Uvv/yCvn37onPnzpaIsUgTeVXzOSgwzekbVHa4hi6XxxZ0Bdn/W+ngZNWSKWvSTkZtpb1RoSBj1wgKhdUTCaNvjc+nn6lCqxB9Vrs8Jhai7W8uRpfVLViwAB988AFef/11ZGRkPxjRyckJffv2xaeffmr2AIu6rDyq+RwUCngpnpi2AitU82mzywOHIWy9ms9qTPh+ba3NlAyM/n1olxBYavuwZOq52CnA9y9rxWH6trGZKnVj2GPMFmbUHpmVlYUjR45gypQpuH//Po4fP47jx4/jwYMHmDdvHjw8cj5vnEwiBFTqXEfPzmuW4mFh+bv5tBXaZIrVfJYhZ5spGej0I5UfKySbNrnZ5UqmSjUG+u+UZ91k04y6hHZ0dETLli1x9uxZlCxZEtWqVbNUXPSMoQ3QC8zKJVNW7bTTmmz9bj5rsWSbqSJQMmXM3XxCwErJpg3+ZuXcF5Taz3wz97bho3LtldF7ZJUqVXD58mVLxEI6hKZJk76G2+ZNpqyTANhlY0tDaFftsWTKfFjNlzdzP05G3yosslQTybkvFOlqfMqN0Xvk5MmT8cEHH2D9+vW4ffs2kpOTJS8yI5H342TssWSq8FbzFa3qqNxZsM2UbZ7WzcrZiLv5FApYJ9m0xf1Z1mTKzL3OS9jgtiaDGJ1it2nTBgDw8ssvS07wQggoFApkZWWZLzp6Xs2n59hhliozKydTdtnY0hDaB9hcH5dCAIzcPkWrms/o37Q12kxZZKkmknNfYMkz6WF0MqXdGzpZXl49oJulykxl3WSq0GLRfza2mTKRkYk4S6asj7910sPovaJx48aWiINyocrj2Xz6OvI0mqZkygYPmPbE3A+cLcwKun2ssl3trFTRKm3KbHB/lvM3xlJo0qPAKfaTJ09w/fp1pKenS4bzDj9zElCp1G2mdMfaY5spi5Pr4MYD7DMG7JO5bZ/8HmpcyJNUhda/hs9k5mRKz3dTyDe78bS3syhCzVqK9HEtf0b/+u7du4d27drBy8sLlStXRs2aNSUvS/rrr7/Qvn17hIeHQ6FQYN26dZLxQgiMGzcOYWFhcHNzQ/PmzXHhwgWLxmRpqjzu5nNyMMNTszSPkykk7QDKNM/+39HFuuvVLvr3CrXuuu2FR3D2/xXaFWx+W0n4K3cy6+KEszsAYGdWdZhWzWeG33CxWqYvo7DTPlaqilAyFdVA7ghsmtFHp6FDhyIxMREHDx6Em5sbNm3ahO+++w5ly5bFb7/9ZokYNR4/fozq1atj7ty5esfPmDEDX375JRYsWICDBw/Cw8MDsbGxSE1NtWhclpTX42TssQG6xUXUye5Ub/g5665XoQCG/A28ewRw9c5/+sIqr33y3UPAW9uBss0NWZCBw6yoQjvgzc1Ap6/NuljFsDNolzYZR0X5Asxs5rtI/UsBA/fhG9+hWostJMcGsymiRXWhVYC3dwMfXJQ7EptkdDXf9u3b8euvv6J27dpwcHBAiRIl0KJFC3h7e2PatGlo27atJeIEALRu3RqtW7fWO04IgS+++AL/+9//0KFDBwDA999/j5CQEKxbtw7du3e3WFyWpC6ZYjWfEcItW0KaK78S8qzXXrj5AcWjCz6/3PuogyMQ+YL5l+vuj9Oi1LM/bOBuvpDKSHQKMM+yLEXOKqeiXO8ZxmY8uTH61/f48WMEB2cX1/v5+eHevXsAgKpVq+LYsWPmjc4IV65cQUJCApo3f37V6+Pjg3r16mH//v25zpeWlma7fWUJofVsPn0lU+ZYh3UfJ0OFnZn7mZKMLwr7qOFJgm4P6JZ6nEwRTh6sju2S7JXRv77y5csjPj4eAFC9enV8/fXXuHnzJhYsWICwsDCzB2iohIQEAEBISIhkeEhIiGacPtOmTYOPj4/mFRERYdE4jSPy7GfKLAe5wlgyRYVTETip+3soDZ62YpiXBR90bOPbWtZ9wXLrLhnI59vaK6N/fUOGDMHt27cBAOPHj8fGjRsRGRmJL7/8ElOnTjV7gJY2evRoJCUlaV43btyQOySJvNpMmWkF2f8zmSJzMHc/U5LRVthHa/TI/r9EfcuvS8vs7jUwuGkZ1Inyy3faPwY3wBsvRGLGK9Ut9jgZSS1aEUhi5baq/wuIiymBIc3LyR0KFZDRbabeeOMNzfvo6Ghcu3YN586dQ2RkJAIDA80anDFCQ7PvoLpz546khOzOnTuoUaNGrvMplUoolYZfDVqbuk9NyyVTLJkiO2GNfTSwLDDyao6H2VpehxrFDJ62crgPJnesmv3HA8t0airJpcy21ELCAsfieqUCUK+UjbdTozwZ/evL+ZBjd3d31KpVS9ZECgBKliyJ0NBQbNu2TTMsOTkZBw8eRExMjIyRmUBo94BuqXUwmSJzsmCbKWud1t387KerEKv0gM5jA1F+jC6ZKlOmDIoXL47GjRujSZMmaNy4McqUKWOJ2HQ8evQIFy8+vy3zypUrOHHiBPz9/REZGYmhQ4di8uTJKFu2LEqWLImxY8ciPDwcHTt2tEp8lpBXD+jmWQEboJOd4D6qy1LJlNbxxiZr+WTtQNIWNwjJzehk6saNG9i5cyd27dqFGTNmoF+/fggPD0fjxo3x0ksv4a233rJEnACAI0eO4KWXXtL8PXz4cABAXFwcli5dihEjRuDx48fo378/EhMT0aBBA2zatAmurq4Wi8nS8no2n1mwZIrMyaJtpngS02Hufqb0r8RCy7VT3A9JD6OTqWLFiqFHjx7o0SO7oeaFCxcwZcoUrFixAitXrrRoMtWkSRNNg2x9FAoFPv74Y3z88ccWi8G6nt/NZ9DvVwjjf+hMpshe8CSmS2H5u/lscrPbZFBUlBmdTD158gR79uzBzp07sXPnThw/fhwVKlTAu+++iyZNmlggxKLNqGo+oTL+kRKF7XEyJDNLtpkiHeZ+nIz+lVhoufaK24N0GZ1M+fr6ws/PDz169MCoUaPQsGFD+PnlfzsvFYy6JM6g3s5VmcYlRUJAc98OS6aI7I+F2kypJOsw22KJCi2jf31t2rRBVlYWVq5ciZUrV2L16tU4f/68JWIrkr7Z/fxuSZVKZVw1nyrTuJVpV5kymSJzMFubKTKIFdpMmeFx6oWLZDuzx3LKZvQZdN26dfjvv/+wadMmxMTEYMuWLWjYsKGmLRUVXGpGFqZsOKv1d6Zx/UwZnUxpPfHcgtUqsZWz+wAL9HSx2DrI/GqXyC5x7lbblp4KQBLav1uz3uFm4wkU7+YjG2N0NZ9a1apVkZmZifT0dKSmpmLz5s1YtWoVVqxYYc74ipRMlZAcI4SA1rP59M9TPtQLuPvsD1WW/olyI7QK8y1YMtW+WhgCPV1QMdS6HSGSaZa+WRfHrz9EjFGdCbLNlFVZoURZwVJronwZ/SuZOXMmXn75ZQQEBKBevXr48ccfUa5cOfzyyy+ahx5TwWSppFdbAvk/TsZZe7DRJVPayZTlGqArFAq8WDoQfh4smbInnkonNCwbBCdHnkxtlqUSUO1+psy3VPORM9lmok96GF0y9eOPP6Jx48bo378/GjZsCB8fH0vEVSTl7PZB4PndfLk+1Fg7gTIpmeIJk8zAkv1MkTyYPBDly+hk6vDhw5aIg/A8cVLzSruDdjtao75LGtxuOAFf6CnZSb75/P03LQpwN98zTKaI6Blh6/1MyYobhHQVqM3U7t278fXXX+PSpUv4+eefUaxYMSxbtgwlS5ZEgwYNzB1jkaHS06jS6+m/8HIAkAkgMZ8FJP9bsBX7RAKOrIIjMzDlzGurbaZKvQRc3gHUsVyHxAXm4vn8vbsZn48qORTZwHdRsT1w9vfnf5dsJF8strRvks0wOpn65Zdf0LNnT/To0QPHjx9HWloaACApKQlTp07Fhg0bzB5kUaHKWTQFINXZF689GoYXSwfiw9jy+md09weePCj4ioPKAw4smSLS641fgEd3AO9wuSPR5eAIjLqRXWXvZL4LIpu74b/r90DKbcA9AEhLATyD5I6ISMLoZGry5MlYsGABevXqhZUrV2qG169fH5MnTzZrcEWNnlwKiR6lcDylLIq5hwERtXKfOaC05QIjsgobbTPl4GibiZSaq4XvkrWFkhgHB8CnWPZ7Z5mftWoL24NsjtHFEfHx8WjUSLeI1cfHB4mJieaIqcjSV82netaWyWIPOiYiykHYUjJLZAeMTqZCQ0Nx8eJFneF79uxBqVKlzBJUUaUvmRJQJ1PWjobIymy1zVQRJN38/C5yJWvnoWRLjE6m+vXrhyFDhuDgwYNQKBS4desWVqxYgQ8++AADBw60RIxFhkqlO0z9U3VgNkVEVsKSKSLjGN1matSoUVCpVGjWrBmePHmCRo0aQalU4oMPPsB7771niRiLjLxLpnhwo8LORttMFUXahyIee4jyZXQypVAoMGbMGHz44Ye4ePEiHj16hEqVKsHT0xNPnz6Fm5ubJeK0ex+uPolMfS3MkX3aaF8jHCX83XXGaUqmeDwjIisRTKCIjFLgZ/O5uLigUqVKAIC0tDTMnDkTM2bMQEJCgtmCK0x+PXkL6Zl66vGeOXb9Ib6Jq6MznCVTVGSwzZSN4ndBlB+Dk6m0tDRMmDABW7duhYuLC0aMGIGOHTtiyZIlGDNmDBwdHTFs2DBLxmrXRraqoLcfqdtJqfh27xWkZqhyqebLluvjZIiIzIzNqomMY3AyNW7cOHz99ddo3rw59u3bh65du6JPnz44cOAAZs6cia5du8LR0XIPy7V3fRuU1Dv89M0kfLv3CgSE/q4ReDcfFRlsM2WTeCFHlC+Dk6nVq1fj+++/x8svv4zTp0+jWrVqyMzMxMmTJ1lqYiZ53c3nyGyKiKyFx3QioxjcNcK///6L6OhoAECVKlWgVCoxbNgwJlImUm8+IXg3HxVxbDNlo/hdEOXH4GQqKysLLi7Pn/3k5OQET0/PPOYgYwjoT6ZuPHwKgOcWKqLYKaIshOABh8gYBlfzCSHQu3dvKJVKAEBqaioGDBgADw8PyXRr1qwxb4SFnOLZVV92yZTu+LSs7P8j/HS7TSAqXHgCt0m8kssDk33KZnAyFRcXJ/n7jTfeMHswRdHz45T+Bui+7i5Y90Z9VCvmY9W4iGxC5U7ArulAQFm5IylS2AM6kXEMTqaWLFliyTiKLO2LPn1dJyidnVAjwtd6ARHJRV8JSHAFYPg5wN3f+vHQM0ysiPJT4E47yTzyq+ZzcDD68YlEhYt3mNwREBHliWdqG5FbA3S2VyAiq5PcWSlfGET2gsmUzJ53jaC/zRTbLhCRtfEmSiLjMJmSmTpVEsit005+RVREsBTWZgj2Rk9kFJ6pZSZpgK63ms96sRAREZHxDGqA/ttvvxm8wJdffrnAwRRN2g3Qc+8Bnajw45WDTWKJIVG+DEqmOnbsaNDCFAoFsrKyTImnyNFuM6WvYMqRd/MRERHZNIOSKZW+xjxkVgJAlp6+EcqHsbNOKiJYAmIzhIJtpgzClvr0DIs9ZKbVAbreaj5XZ3YFRkTWxfSJyDgFOlM/fvwYu3btwvXr15Geni4ZN3jwYLMEVlQonl0BZvczpXcCq8ZDJB/u67ZCcjcfj0FE+TI6mTp+/DjatGmDJ0+e4PHjx/D398d///0Hd3d3BAcHM5kykvZhSrDTTiIiIrtjdDXfsGHD0L59ezx8+BBubm44cOAArl27hujoaHz22WeWiLFQ026AnqU3mWJNLBURvHCwGexnisg4Rp+pT5w4gffffx8ODg5wdHREWloaIiIiMGPGDHz00UeWiNFoc+fORVRUFFxdXVGvXj0cOnRI7pDylWs1Hw9kRERENs3oZMrZ2Vnz8N3g4GBcv34dAODj44MbN26YN7oCWLVqFYYPH47x48fj2LFjqF69OmJjY3H37l25Q9NL+0HH+qv5WDJFBWVvdxrxwsFWSPYclhgS5cvoM3XNmjVx+PBhAEDjxo0xbtw4rFixAkOHDkWVKlXMHqCxZs6ciX79+qFPnz6oVKkSFixYAHd3d3z77bdyh6aXppoP+p/NxwMZERGRbTM6mZo6dSrCwsIAAFOmTIGfnx8GDhyIe/fu4euvvzZ7gMZIT0/H0aNH0bx5c80wBwcHNG/eHPv375cxMsNk6evOiyVTVFTwwsF2sJ8pIqMYfTdf7dq1Ne+Dg4OxadMmswZkiv/++w9ZWVkICQmRDA8JCcG5c+f0zpOWloa0tDTN38nJyRaNMafnDdBzeTYfD2REZG32VkNMJDOjiz2aNm2KxMREneHJyclo2rSpOWKyqmnTpsHHx0fzioiIkCUOAbaZoqKOFw62gv1MGYpZJ2Uz+ky9c+dOnY46ASA1NRW7d+82S1AFFRgYCEdHR9y5c0cy/M6dOwgNDdU7z+jRo5GUlKR5WbsRveJ5o6lcqvl4ICMiIrJlBlfz/f3335r3//zzDxISEjR/Z2VlYdOmTShWrJh5ozOSi4sLoqOjsW3bNs3DmVUqFbZt24Z3331X7zxKpRJKpdKKUUqpU6XcG6CzZIqKCF442Az2M0VkHIOTqRo1akChUEChUOitznNzc8NXX31l1uAKYvjw4YiLi0Pt2rVRt25dfPHFF3j8+DH69Okjb2C/DQZUWTqDfTMyMdopBQtFR/3VfERERGTTDE6mrly5AiEESpUqhUOHDiEoKEgzzsXFBcHBwXB0dLRIkMZ49dVXce/ePYwbNw4JCQmoUaMGNm3apNMo3epOrgSy0nQGuwN42wm4nhkGlairOx9LpqjIYAmITWKJIVG+DE6mSpQoASC72szWvfvuu7lW68mm6f8AoVsylXFyNZzvnYEr0pClrwt0HsiIiIhsmtFdIwDApUuX8MUXX+Ds2bMAgEqVKmHIkCEoXbq0WYMrVOrrfwB05r9/w/neGSjYZoqKOl442Ay2mSIyjtFn6s2bN6NSpUo4dOgQqlWrhmrVquHgwYOoXLkytm7daokYCzWFdhN0PpuPiIjI7hhdMjVq1CgMGzYM06dP1xk+cuRItGjRwmzBFQnPrsYVUHfamSOjYskUFRm8cLAVQsF+poiMYfSZ+uzZs+jbt6/O8DfffBP//POPWYIqUrS6QM/is/mIiIjsjtHJVFBQEE6cOKEz/MSJEwgODjZHTEWMumQqu5pPwZIpKqp44WCj+L0Q5cfgar6PP/4YH3zwAfr164f+/fvj8uXLePHFFwEAe/fuxSeffILhw4dbLNBC69kJxAECWSqh57DFAxkREZEtMziZmjhxIgYMGICxY8fCy8sLn3/+OUaPHg0ACA8Px4QJEzB4sP471igvz9tMZaoES6aoCOOFg63gs/kMxI6W6RmDkyl179wKhQLDhg3DsGHDkJKSAgDw8vKyTHRFgMIhO1lSQCBD38P5eCAjIiKyaUbdzafIcWJnEmUOz7tGyMxS6SmZYjJFRQT3dZvBfqaIjGNUMlWuXDmdhCqnBw8emBRQkaPVNUJGFttMERER2RujkqmJEyfCx8fHUrEUSQqtu/n0V/OxzRQVFbxwsB1sM0VkDKOSqe7du7P7AzMTCmkyxWo+IiIi+2JwsUd+1XtUQDrVfDnvDuF2pyKCxxibwXvUiIxjcDIleAuoRSi0SqbS9VXz8bBGRERk0wyu5lOp9J3oyXTPkilF9t18OgS3OxURLJmyIWwzRWQMtm6WW37VfCwRJCIismlMpmT3vJ8pvQ3QWc1HRFbGfqaIjMNkSmaSNlOZKt3DFnMpIpITq/mI8sVkSm5a1XyZKj2ZE9tMEZGV6es+mIhyx2RKZtqddup9nAyLpohIVkysiPLDZEpuz0qmXnQ4A++Me2jlcFg6niVTRGRtzJ+IjGJUD+hkAc+SqVoOF7EsqTfgkmM87+ajggquBKTcljsKskPF/dyBhGd/sM1UHnh8pmwsmZJdfgcq/lipgDrOB6J7A2//JXckZGfealRK7hCI7ApLpuSW31UfS6aooLxCgPaz5Y6C7JCXUvvUwJIpovywZEpmCkU+XwHbTBGR1TGBIjIGkym55dsegSVTRCQjtpkiyheTKZkp8rsCZDUfEVkbEygiozCZkhtLpojIpjGxIsoPkym5sc0UEdkcJlBExmAyJTNW8xGRTWOVH1G+mEzJjV0jEJGtYQJFZBQmUzJTsM0UEdk0Jla54uGZnmEyJTeWTBGRzWECRWQMJlOyY8kUEdkwVvkR5YvJlNzyLZni3XxEZGVMoIiMwmRKdqzmIyJbxsSKKD9MpuTGBuhEZHOYQBEZg8mU7FgyRUQ2jFV+RPmym2RqypQpePHFF+Hu7g5fX1+901y/fh1t27aFu7s7goOD8eGHHyIzM9O6gRqLbaaIyNYwgSIyipPcARgqPT0dXbt2RUxMDBYvXqwzPisrC23btkVoaCj27duH27dvo1evXnB2dsbUqVNliNhQ+Ry0gspbJwwiIr2YWBHlx25KpiZOnIhhw4ahatWqesdv2bIF//zzD5YvX44aNWqgdevWmDRpEubOnYv09HQrR2uEvI5TTT4C6rxltVCIiLIxgSIyht0kU/nZv38/qlatipCQEM2w2NhYJCcn48yZM7nOl5aWhuTkZMnLuvI4aDUZCTg6Wy8UIqKcWOWXB7ZppWyFJplKSEiQJFIANH8nJCTkOt+0adPg4+OjeUVERFg0Th08UBGRreFxicgosiZTo0aNgkKhyPN17tw5i8YwevRoJCUlaV43btyw6Pp0KApNPktEhRITK6L8yNoA/f3330fv3r3znKZUqVIGLSs0NBSHDh2SDLtz545mXG6USiWUSqVB67AMHqiIyNbY6XHJ2UPuCKiIkjWZCgoKQlBQkFmWFRMTgylTpuDu3bsIDg4GAGzduhXe3t6oVKmSWdZhESxOJyJbZg/HqDd+ATaOAjrOlzsSKqLspmuE69ev48GDB7h+/TqysrJw4sQJAECZMmXg6emJli1bolKlSujZsydmzJiBhIQE/O9//8OgQYNkLnnKjx0cqIioaLGHBEpbmebAe0fkjoKKMLtJpsaNG4fvvvtO83fNmjUBADt27ECTJk3g6OiI9evXY+DAgYiJiYGHhwfi4uLw8ccfyxWyYeztoEVEREQSdpNMLV26FEuXLs1zmhIlSmDDhg3WCchsmEwRka3hcYnIGLyVTG4smSIiW8ZjFFG+mEzJjgcqIrIxTKCIjMJkSm76DlqNRwEjr1o9FCIiIjIekynZ6UmmfCMANz/rh0JEBIAl5kTGYTIlN30lU+wVnYjI9gk+m4+y8awtOyZTRGRj2GaKyCg8a8tN70GLBzIiIiJ7wWTKFrFkioiIyG7wrC03vW2mWDJFRDLiMYjIKEymZMc2U0RERPaMZ2258W4+IrI5LJkiMgbP2nLTlzixiJ2IiMhuMJmSHUumiMjG8IKOyCg8a8uN1XxERER2jWdt2TGZIiJbw5Ipw7AHdMrGs7bc2GknERGRXWMyJTuWTBGRjWGbKSKj8KwtN7aZIiIisms8a8uOPaATka3hMYjIGEym5MbHyRAREdk1JlOyYzUfEdkYXtARGYVnbbmxzRQREZFd41lbdkymiMjWsGSKyBg8a8uNJVNERER2jWdtIiKSYpspwwj2gE7ZmEzJjQctIiIiu8ZkSnZ6kile7RCRrHiRR2QMJlNyY8kUERGRXWMyJTt9yRRLpohIRrzIIzIKkym56btzT6isHwcREREVCJMpufEKkIhsDo9LRMZgMiU7NkAnIiKyZ0ym5MaSKSKyNTwuERmFyZRNYskUERGRvWAyJTd9VXpsgE5EsmLJlGF44UvZmEzJjj9GIiIie8ZkSm56S6aYYBGRjNhmisgodpFMXb16FX379kXJkiXh5uaG0qVLY/z48UhPT5dM9/fff6Nhw4ZwdXVFREQEZsyYIVPExtCXODGZIiIishdOcgdgiHPnzkGlUuHrr79GmTJlcPr0afTr1w+PHz/GZ599BgBITk5Gy5Yt0bx5cyxYsACnTp3Cm2++CV9fX/Tv31/mT5AHfaVQJRpYPw4iIg2WTBEZwy6SqVatWqFVq1aav0uVKoX4+HjMnz9fk0ytWLEC6enp+Pbbb+Hi4oLKlSvjxIkTmDlzpm0nUzlLoRoMA5xd5QmFiIiIjGYX1Xz6JCUlwd/fX/P3/v370ahRI7i4uGiGxcbGIj4+Hg8fPsx1OWlpaUhOTpa8rCpnyZTC0brrJyLKiW2miIxil8nUxYsX8dVXX+Htt9/WDEtISEBISIhkOvXfCQkJuS5r2rRp8PHx0bwiIiIsE3Su2D6KiIjInsmaTI0aNQoKhSLP17lz5yTz3Lx5E61atULXrl3Rr18/k2MYPXo0kpKSNK8bN26YvEyj8M49IrI1LJkiMoqsbabef/999O7dO89pSpUqpXl/69YtvPTSS3jxxRexcOFCyXShoaG4c+eOZJj679DQ0FyXr1QqoVQqjYzcgngQIyIisiuyJlNBQUEICgoyaNqbN2/ipZdeQnR0NJYsWQIHB2mhWkxMDMaMGYOMjAw4OzsDALZu3Yry5cvDz8/P7LGbDUumiIjsE4/f9IxdtJm6efMmmjRpgsjISHz22We4d+8eEhISJG2hXn/9dbi4uKBv3744c+YMVq1ahdmzZ2P48OEyRm4I/hiJiIjsmV10jbB161ZcvHgRFy9eRPHixSXjxLMrAx8fH2zZsgWDBg1CdHQ0AgMDMW7cOBvvFgG8siEiIrJzdpFM9e7dO9+2VQBQrVo17N692/IBWRTbTBEREdkTu6jmK9xYMkVERGTPmEzJTajkjoCIiIhMwGRKbmwzRUREZNeYTMku5+Nk2GaKiIjInjCZkhtLpoiIiOwakynZMZkiIiKyZ0ym5MaSKSIiIrvGZEp2OZMptpkiIiKyJ0ym5MaSKSIiO8XjN2VjMiU7/hiJiIjsGZMpuRWLljsCIiIiMgGTKbmFVUe3tLHP/2Y/U0RERHaFyZQNOCQqyh0CERERFRCTKSIiIiITMJmyAYt61ZY7BCIiIiogJlM2oEWlEK2/2GaKiIjInjCZIiIiIjIBkykiIiIiEzCZIiIiKgg+wYKeYTJla9jPFBERkV1hMkVERLlj6QtRvphMEREREZmAyRQRERGRCZhM2Ry2mSIiIrInTKaIiIiITMBkioiIiMgETKaIiIiITMBkioiIiMgETKaIiIgKhH1wUTYmU0REREQmYDJFRES54yOu8sBtQ9mYTNka/jaJiIjsCpMpIiIiIhMwmSIiIioQNkCnbEymiIiIiEzAZMrmsNEUERGRPWEyRURERGQCJlNEREREJrCbZOrll19GZGQkXF1dERYWhp49e+LWrVuSaf7++280bNgQrq6uiIiIwIwZM2SK1gSOznJHQGQeTq4Fn9fRxXxxkGkUjnJHYLuc3eSOgGyE3SRTL730En766SfEx8fjl19+waVLl/DKK69oxicnJ6Nly5YoUaIEjh49ik8//RQTJkzAwoULZYzaCI1GAMGVgeg+ckdCZJouiwH/UkDnRcbP22o6EFAWaPo/88dFxqn7NhBeCyjfWu5IbM9rKwG/kkCPX+SOhGyEQghhl/d2/vbbb+jYsSPS0tLg7OyM+fPnY8yYMUhISICLS/ZV7ahRo7Bu3TqcO3fO4OUmJyfDx8cHSUlJ8Pb2tlT4REREZEZynr/tpmRK24MHD7BixQq8+OKLcHbOrhbbv38/GjVqpEmkACA2Nhbx8fF4+PBhrstKS0tDcnKy5EVERERkKLtKpkaOHAkPDw8EBATg+vXr+PXXXzXjEhISEBISIple/XdCQkKuy5w2bRp8fHw0r4iICMsET0RERIWSrMnUqFGjoFAo8nxpV9F9+OGHOH78OLZs2QJHR0f06tULptZSjh49GklJSZrXjRs3TP1YREREVIQ4ybny999/H717985zmlKlSmneBwYGIjAwEOXKlUPFihURERGBAwcOICYmBqGhobhz545kXvXfoaGhuS5fqVRCqVQW/EMQERFRkSZrMhUUFISgoKACzatSqQBkt3kCgJiYGIwZMwYZGRmadlRbt25F+fLl4efnZ56AiYiIiHKwizZTBw8exJw5c3DixAlcu3YN27dvx2uvvYbSpUsjJiYGAPD666/DxcUFffv2xZkzZ7Bq1SrMnj0bw4cPlzl6IiIiKszsIplyd3fHmjVr0KxZM5QvXx59+/ZFtWrVsGvXLk0VnY+PD7Zs2YIrV64gOjoa77//PsaNG4f+/fvLHD0REREVZnbbz5SlsJ8pIiIi+8N+poiIiIjsFJMpIiIiIhMwmSIiIiIyAZMpIiIiIhMwmSIiIiIyAZMpIiIiIhPI2gO6LVL3FJGcnCxzJERERGQo9Xlbjh6fmEzlkJKSAgCIiIiQORIiIiIyVkpKCnx8fKy6TnbamYNKpcKtW7fg5eUFhUJhtuUmJycjIiICN27cYGegFsTtbD3c1tbB7Wwd3M7WYcntLIRASkoKwsPD4eBg3VZMLJnKwcHBAcWLF7fY8r29vflDtQJuZ+vhtrYObmfr4Ha2DkttZ2uXSKmxAToRERGRCZhMEREREZmAyZSVKJVKjB8/HkqlUu5QCjVuZ+vhtrYObmfr4Ha2jsK6ndkAnYiIiMgELJkiIiIiMgGTKSIiIiITMJkiIiIiMgGTKSIiIiITMJmykrlz5yIqKgqurq6oV68eDh06JHdIdmPatGmoU6cOvLy8EBwcjI4dOyI+Pl4yTWpqKgYNGoSAgAB4enqiS5cuuHPnjmSa69evo23btnB3d0dwcDA+/PBDZGZmWvOj2JXp06dDoVBg6NChmmHczuZz8+ZNvPHGGwgICICbmxuqVq2KI0eOaMYLITBu3DiEhYXBzc0NzZs3x4ULFyTLePDgAXr06AFvb2/4+vqib9++ePTokbU/is3KysrC2LFjUbJkSbi5uaF06dKYNGmS5Nlt3M7G++uvv9C+fXuEh4dDoVBg3bp1kvHm2qZ///03GjZsCFdXV0RERGDGjBmW/mgFJ8jiVq5cKVxcXMS3334rzpw5I/r16yd8fX3FnTt35A7NLsTGxoolS5aI06dPixMnTog2bdqIyMhI8ejRI800AwYMEBEREWLbtm3iyJEj4oUXXhAvvviiZnxmZqaoUqWKaN68uTh+/LjYsGGDCAwMFKNHj5bjI9m8Q4cOiaioKFGtWjUxZMgQzXBuZ/N48OCBKFGihOjdu7c4ePCguHz5sti8ebO4ePGiZprp06cLHx8fsW7dOnHy5Enx8ssvi5IlS4qnT59qpmnVqpWoXr26OHDggNi9e7coU6aMeO211+T4SDZpypQpIiAgQKxfv15cuXJFrF69Wnh6eorZs2drpuF2Nt6GDRvEmDFjxJo1awQAsXbtWsl4c2zTpKQkERISInr06CFOnz4tfvzxR+Hm5ia+/vpra31MozCZsoK6deuKQYMGaf7OysoS4eHhYtq0aTJGZb/u3r0rAIhdu3YJIYRITEwUzs7OYvXq1Zppzp49KwCI/fv3CyGyf/wODg4iISFBM838+fOFt7e3SEtLs+4HsHEpKSmibNmyYuvWraJx48aaZIrb2XxGjhwpGjRokOt4lUolQkNDxaeffqoZlpiYKJRKpfjxxx+FEEL8888/AoA4fPiwZpqNGzcKhUIhbt68abng7Ujbtm3Fm2++KRnWuXNn0aNHDyEEt7M55EymzLVN582bJ/z8/CTHjZEjR4ry5ctb+BMVDKv5LCw9PR1Hjx5F8+bNNcMcHBzQvHlz7N+/X8bI7FdSUhIAwN/fHwBw9OhRZGRkSLZxhQoVEBkZqdnG+/fvR9WqVRESEqKZJjY2FsnJyThz5owVo7d9gwYNQtu2bSXbE+B2NqfffvsNtWvXRteuXREcHIyaNWti0aJFmvFXrlxBQkKCZFv7+PigXr16km3t6+uL2rVra6Zp3rw5HBwccPDgQet9GBv24osvYtu2bTh//jwA4OTJk9izZw9at24NgNvZEsy1Tffv349GjRrBxcVFM01sbCzi4+Px8OFDK30aw/FBxxb233//ISsrS3JyAYCQkBCcO3dOpqjsl0qlwtChQ1G/fn1UqVIFAJCQkAAXFxf4+vpKpg0JCUFCQoJmGn3fgXocZVu5ciWOHTuGw4cP64zjdjafy5cvY/78+Rg+fDg++ugjHD58GIMHD4aLiwvi4uI020rfttTe1sHBwZLxTk5O8Pf357Z+ZtSoUUhOTkaFChXg6OiIrKwsTJkyBT169AAAbmcLMNc2TUhIQMmSJXWWoR7n5+dnkfgLiskU2ZVBgwbh9OnT2LNnj9yhFDo3btzAkCFDsHXrVri6usodTqGmUqlQu3ZtTJ06FQBQs2ZNnD59GgsWLEBcXJzM0RUeP/30E1asWIEffvgBlStXxokTJzB06FCEh4dzO5NZsZrPwgIDA+Ho6Khzx9OdO3cQGhoqU1T26d1338X69euxY8cOFC9eXDM8NDQU6enpSExMlEyvvY1DQ0P1fgfqcZRdjXf37l3UqlULTk5OcHJywq5du/Dll1/CyckJISEh3M5mEhYWhkqVKkmGVaxYEdevXwfwfFvlddwIDQ3F3bt3JeMzMzPx4MEDbutnPvzwQ4waNQrdu3dH1apV0bNnTwwbNgzTpk0DwO1sCebapvZ2LGEyZWEuLi6Ijo7Gtm3bNMNUKhW2bduGmJgYGSOzH0IIvPvuu1i7di22b9+uU/QbHR0NZ2dnyTaOj4/H9evXNds4JiYGp06dkvyAt27dCm9vb52TWlHVrFkznDp1CidOnNC8ateujR49emjeczubR/369XW69zh//jxKlCgBAChZsiRCQ0Ml2zo5ORkHDx6UbOvExEQcPXpUM8327duhUqlQr149K3wK2/fkyRM4OEhPc46OjlCpVAC4nS3BXNs0JiYGf/31FzIyMjTTbN26FeXLl7e5Kj4A7BrBGlauXCmUSqVYunSp+Oeff0T//v2Fr6+v5I4nyt3AgQOFj4+P2Llzp7h9+7bm9eTJE800AwYMEJGRkWL79u3iyJEjIiYmRsTExGjGq2/Zb9mypThx4oTYtGmTCAoK4i37+dC+m08IbmdzOXTokHBychJTpkwRFy5cECtWrBDu7u5i+fLlmmmmT58ufH19xa+//ir+/vtv0aFDB723l9esWVMcPHhQ7NmzR5QtW7ZI37KfU1xcnChWrJima4Q1a9aIwMBAMWLECM003M7GS0lJEcePHxfHjx8XAMTMmTPF8ePHxbVr14QQ5tmmiYmJIiQkRPTs2VOcPn1arFy5Uri7u7NrhKLuq6++EpGRkcLFxUXUrVtXHDhwQO6Q7AYAva8lS5Zopnn69Kl45513hJ+fn3B3dxedOnUSt2/flizn6tWronXr1sLNzU0EBgaK999/X2RkZFj509iXnMkUt7P5/P7776JKlSpCqVSKChUqiIULF0rGq1QqMXbsWBESEiKUSqVo1qyZiI+Pl0xz//598dprrwlPT0/h7e0t+vTpI1JSUqz5MWxacnKyGDJkiIiMjBSurq6iVKlSYsyYMZLb7bmdjbdjxw69x+S4uDghhPm26cmTJ0WDBg2EUqkUxYoVE9OnT7fWRzSaQgitrmCJiIiIyChsM0VERERkAiZTRERERCZgMkVERERkAiZTRERERCZgMkVERERkAiZTRERERCZgMkVERERkAiZTRFQoXL16FQqFAidOnLDYOnr37o2OHTtabPlEZJ+YTBGRTejduzcUCoXOq1WrVgbNHxERgdu3b6NKlSoWjpSISMpJ7gCIiNRatWqFJUuWSIYplUqD5nV0dLTJp8kTUeHHkikishlKpRKhoaGSl/oJ8QqFAvPnz0fr1q3h5uaGUqVK4eeff9bMm7Oa7+HDh+jRoweCgoLg5uaGsmXLShK1U6dOoWnTpnBzc0NAQAD69++PR48eacZnZWVh+PDh8PX1RUBAAEaMGIGcT99SqVSYNm0aSpYsCTc3N1SvXl0SExEVDUymiMhujB07Fl26dMHJkyfRo0cPdO/eHWfPns112n/++QcbN27E2bNnMX/+fAQGBgIAHj9+jNjYWPj5+eHw4cNYvXo1/vzzT7z77rua+T///HMsXboU3377Lfbs2YMHDx5g7dq1knVMmzYN33//PRYsWIAzZ85g2LBheOONN7Br1y7LbQQisj0yP2iZiEgIIURcXJxwdHQUHh4ekteUKVOEEEIAEAMGDJDMU69ePTFw4EAhhBBXrlwRAMTx48eFEEK0b99e9OnTR++6Fi5cKPz8/MSjR480w/744w/h4OAgEhIShBBChIWFiRkzZmjGZ2RkiOLFi4sOHToIIYRITU0V7u7uYt++fZJl9+3bV7z22msF3xBEZHfYZoqIbMZLL72E+fPnS4b5+/tr3sfExEjGxcTE5Hr33sCBA9GlSxccO3YMLVu2RMeOHfHiiy8CAM6ePYvq1avDw8NDM339+vWhUqkQHx8PV1dX3L59G/Xq1dOMd3JyQu3atTVVfRcvXsSTJ0/QokULyXrT09NRs2ZN4z88EdktJlNEZDM8PDxQpkwZsyyrdevWuHbtGjZs2ICtW7eiWbNmGDRoED777DOzLF/dvuqPP/5AsWLFJOMMbTRPRIUD20wRkd04cOCAzt8VK1bMdfqgoCDExcVh+fLl+OKLL7Bw4UIAQMWKFXHy5Ek8fvxYM+3evXvh4OCA8uXLw8fHB2FhYTh48KBmfGZmJo4ePar5u1KlSlAqlbh+/TrKlCkjeUVERJjrIxORHWDJFBHZjLS0NCQkJEiGOTk5aRqOr169GrVr10aDBg2wYsUKHDp0CIsXL9a7rHHjxiE6OhqVK1dGWloa1q9fr0m8evTogfHjxyMuLg4TJkzAvXv38N5776Fnz54ICQkBAAwZMgTTp09H2bJlUaFCBcycOROJiYma5Xt5eeGDDz7AsGHDoFKp0KBBAyQlJWHv3r3w9vZGXFycBbYQEdkiJlNEZDM2bdqEsLAwybDy5cvj3LlzAICJEydi5cqVeOeddxAWFoYff/wRlSpV0rssFxcXjB49GlevXoWbmxsaNmyIlStXAgDc3d2xefNmDBkyBHXq1IG7uzu6dOmCmTNnauZ///33cfv2bcTFxcHBwQFvvvkmOnXqhKSkJM00kyZNQlBQEKZNm4bLly/D19cXtWrVwkcffWTuTUNENkwhRI6OU4iIbJBCocDatWv5OBcisjlsM0VERERkAiZTRERERCZgmykisgtskUBEtoolU0REREQmYDJFREREZAImU0REREQmYDJFREREZAImU0REREQmYDJFREREZAImU0REREQmYDJFREREZAImU0REREQm+D/ZM33naoTS5QAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(q_rewards, label='Q-Learning')\n",
    "plt.plot(double_q_rewards, label='Double Q-learning')\n",
    "plt.xlabel('Episode')\n",
    "plt.ylabel('Total Reward')\n",
    "plt.legend()\n",
    "plt.title('Comparison of Q-Learning and Double Q-learning - Deterministic')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}