connect4-rl1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using gpu device 0: GeForce GTX 980 (CNMeM is enabled with initial size: 75.0% of memory, CuDNN 4007)\n",
      "/usr/lib/python3.4/site-packages/theano/tensor/signal/downsample.py:5: UserWarning: downsample module has been moved to the pool module.\n",
      "  warnings.warn(\"downsample module has been moved to the pool module.\")\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'float32'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import theano\n",
    "import theano.tensor as T\n",
    "\n",
    "import lasagne\n",
    "floatX = theano.config.floatX\n",
    "floatX"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from IPython.display import HTML, display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%load_ext Cython"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%%cython\n",
    "# cython: infer_types=True, annotation_typing=True\n",
    "## cython: infer_types.verbose=True \n",
    "from IPython.display import HTML, display\n",
    "import numpy as np\n",
    "\n",
    "floatX = np.float32\n",
    "\n",
    "binary6 = np.array([ list(map(int,bin(2**6+i)[:2:-1])) for i in range(2**6)], dtype=floatX)\n",
    "height = np.array([-1]*65, dtype=np.int32)\n",
    "for __i in range(6):\n",
    "    height[2**__i]=__i\n",
    "\n",
    "cdef class Connect4:\n",
    "    cdef public:\n",
    "        long turn\n",
    "        long long[2] data\n",
    "    cpdef long get_col_row(self, col: long, row: long):\n",
    "        pos = col * 7 + row\n",
    "        mask = (<long long>1) << pos \n",
    "        if self.data[1] & mask:\n",
    "            return 2\n",
    "        return bool(self.data[0] & mask)\n",
    "    \n",
    "    cpdef long is_end(self):\n",
    "        cdef long long mask\n",
    "        bitboard = self.data[1-self.turn%2]\n",
    "        bound = (<long long>1)<<48 # 49 = 7*(6+1)  \n",
    "        # horizontal: 0x204081 = 1|(1<<7)|(1<<14)|(1<<21)\n",
    "        # vertical: 0xf = 1|(1<<1)|(1<<2)|(1<<3)\n",
    "        # up-right: 0x1010101 = 1|(1<<8)|(1<<16)|(1<<24)\n",
    "        # down-right: 0x208208 = (1<<3)|(1<<9)|(1<<15)|(1<<21)\n",
    "        for mask in [0x204081, 0xf,  0x1010101, 0x208208]:\n",
    "            while mask < bound:\n",
    "                if mask & bitboard == mask:\n",
    "                    return True\n",
    "                mask <<= 1\n",
    "        return False\n",
    "    \n",
    "    cpdef set_col_row(self, col:long, row:long, value:long):\n",
    "        # assert value in [0,1,2]\n",
    "        pos = col * 7 + row\n",
    "        mask = (<long long>1) << pos\n",
    "        neg_mask = ~mask       \n",
    "        if value == 1 or value ==2:\n",
    "            self.data[value-1] |= mask\n",
    "            self.data[2-value] &= neg_mask\n",
    "        else:\n",
    "            self.data[0] &= neg_mask\n",
    "            self.data[1] &= neg_mask\n",
    "            \n",
    "    def __init__(self, data=None, turn=0):\n",
    "        if data is not None:\n",
    "            self.data = data[:]\n",
    "        else:\n",
    "            self.data = [0, 0]\n",
    "        self.turn = turn\n",
    "        \n",
    "    cpdef remove(self, col:long):\n",
    "        shift = col*7\n",
    "        mask = (((self.data[0]|self.data[1]) >> shift) &0x3f) +1\n",
    "        mask = (mask >> 1) << shift\n",
    "        # print(shift, hex(mask), hex(self.data[0]), hex(self.data[1]))\n",
    "        neg_mask = ~mask\n",
    "        self.data[0] &= neg_mask\n",
    "        self.data[1] &= neg_mask\n",
    "    \n",
    "    def _np_branch(self):\n",
    "        c = self.turn%2 # who's turn\n",
    "        base = np.zeros((2,7,6), dtype=floatX)\n",
    "        pos = []\n",
    "        moves = []\n",
    "        red, yellow = self.data\n",
    "        for i in range(7):\n",
    "            mask = ((red|yellow) &0x3f) + 1\n",
    "            p = height[mask]\n",
    "            if p != -1:\n",
    "                moves.append(i)\n",
    "                pos.append(height[mask])\n",
    "            base[c, i] = binary6[red&0x3f]\n",
    "            base[1-c, i] = binary6[yellow&0x3f]\n",
    "            red >>= 7\n",
    "            yellow >>= 7\n",
    "        boards = np.zeros( (len(moves), 2, 7, 6), dtype=floatX)\n",
    "        for i in range(len(moves)):\n",
    "            m = moves[i]\n",
    "            p = pos[i]\n",
    "            boards[i]=base\n",
    "            boards[i, 0, m, p] = 1\n",
    "        return moves, boards\n",
    "    \n",
    "    def _np_board(self):\n",
    "        c = (self.turn-1)%2 # who played\n",
    "        board = np.zeros((2, 7, 6), dtype=floatX)\n",
    "        pos = []\n",
    "        moves = []\n",
    "        red, yellow = self.data\n",
    "        for i in range(7):\n",
    "            mask = ((red|yellow) &0x3f) + 1\n",
    "            p = height[mask]\n",
    "            if p != -1:\n",
    "                moves.append(i)\n",
    "                pos.append(height[mask])\n",
    "            board[c, i] = binary6[red&0x3f]\n",
    "            board[1-c, i] = binary6[yellow&0x3f]\n",
    "            red >>= 7\n",
    "            yellow >>= 7\n",
    "        return board\n",
    "    \n",
    "        \n",
    "    cpdef move(self, col:long, test=False):\n",
    "        # assert 0<= col <7\n",
    "        shift = col*7\n",
    "        mask = (((self.data[0]|self.data[1]) >> shift) &0x3f) +1\n",
    "        # print(\"mask=\", mask)\n",
    "        if mask >= 64:\n",
    "            return None\n",
    "        if not test:\n",
    "            self.data[self.turn%2] |= (mask<<shift)\n",
    "            self.turn += 1\n",
    "        return self\n",
    "    \n",
    "    def board_data(self):\n",
    "        for i in range(7):\n",
    "            for j in range(6):\n",
    "                c = self.get_col_row(i,j)\n",
    "                if c!=0:\n",
    "                    yield i,j,c\n",
    "                    \n",
    "    def _repr_html_(self):\n",
    "        def pos(i):\n",
    "            return int(7+(220-6.5)*i/8)\n",
    "        imgstr = \"<img src='img/%s.png' width='23px' height='23px' style='position: absolute; top: %spx; left: %spx;margin-top: 0;z-index: %d' />\"\n",
    "        header = \"\"\"<div style=\"width: 200px; height:180px;position: relative;background: blue\">\"\"\"\n",
    "        header += \"\\n\".join(imgstr%('empty', pos(5-j), pos(i), 0) for i in range(7) for j in range(6))\n",
    "        return header +\"\\n\".join(imgstr%('red_coin' if c==1 else 'yellow_coin', pos(5-j), pos(i), 2) for (i,j,c) in self.board_data()) +\"</div>\"\n",
    "    \n",
    "    def display(self):\n",
    "        display(HTML(self._repr_html_()))\n",
    "    \n",
    "    def __repr__(self):\n",
    "        row_str = lambda j: \"\".join(\".ox\"[self.get_col_row(i,j)] for i in range(7))\n",
    "        return \"\\n\".join(row_str(j) for j in range(5,-1,-1))\n",
    "\n",
    "from random import randint\n",
    "def random_play(init_data=None, init_turn=0, display=False):\n",
    "    game = Connect4(init_data, init_turn)\n",
    "    while game.turn < 42 and not game.is_end():\n",
    "        while game.move(randint(0,6)) is None:\n",
    "            continue\n",
    "    if display:\n",
    "        game.display()\n",
    "    if game.is_end():\n",
    "        return game.turn\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def MC_agent(_game, N=200):\n",
    "    score = [-1.0*N]*7\n",
    "    for i in range(7):\n",
    "        game = Connect4(_game.data, _game.turn)\n",
    "        if game.move(i):\n",
    "            if game.is_end():\n",
    "                return i\n",
    "            s = 0\n",
    "            for j in range(N):\n",
    "                #print(\"move\", i, \"case\", j)\n",
    "                r = random_play(game.data, game.turn)\n",
    "                turn = (r-1)%2\n",
    "                if r == 0:\n",
    "                    pass\n",
    "                elif  (r-1)%2 == _game.turn%2:\n",
    "                    s += 0.95** (r-_game.turn-1)\n",
    "                else:\n",
    "                    s -= .95** (r-_game.turn-1)\n",
    "            score[i] = s/N\n",
    "    return max(zip(score, range(7)))[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def random_vs_MC(init_data=None, init_turn=0, display=False):\n",
    "    game = Connect4(init_data, init_turn)\n",
    "    while game.turn < 42 and not game.is_end():\n",
    "        if game.turn%2 == 0:\n",
    "            while game.move(randint(0,6)) is None:\n",
    "                continue\n",
    "        else:\n",
    "            i = MC_agent(game)\n",
    "            game.move(i)\n",
    "        if display == 'all':\n",
    "            game.display()\n",
    "    if display:\n",
    "        game.display()\n",
    "    if game.is_end():\n",
    "        return game.turn\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "input_var = T.tensor4('inputs')\n",
    "target_var = T.vector('targets')\n",
    "l_in = lasagne.layers.InputLayer(shape=(None, 2, 7, 6), input_var=input_var)\n",
    "#l_in_drop = lasagne.layers.DropoutLayer(l_in, p=0.2)\n",
    "l_hidden = lasagne.layers.DenseLayer(l_in, num_units=400, nonlinearity=lasagne.nonlinearities.rectify, W=lasagne.init.GlorotUniform())\n",
    "l_hidden2 = lasagne.layers.DenseLayer(l_hidden, num_units=200, nonlinearity=lasagne.nonlinearities.rectify, W=lasagne.init.GlorotUniform())\n",
    "l_hidden2_drop = lasagne.layers.DropoutLayer(l_hidden2, p=0.2)\n",
    "l_hidden3 = lasagne.layers.DenseLayer(l_hidden2_drop, num_units=40, nonlinearity=lasagne.nonlinearities.rectify, W=lasagne.init.GlorotUniform())\n",
    "l_out = lasagne.layers.DenseLayer(l_hidden3, num_units=1, nonlinearity=lasagne.nonlinearities.tanh, W=lasagne.init.GlorotUniform())\n",
    "\n",
    "prediction = lasagne.layers.get_output(l_out).flatten()\n",
    "V = theano.function([input_var], prediction)\n",
    "#loss = lasagne.objectives.binary_crossentropy(prediction, target_var)\n",
    "loss = lasagne.objectives.squared_error(prediction, target_var)\n",
    "\n",
    "loss = loss.mean()\n",
    "\n",
    "params = lasagne.layers.get_all_params(l_out, trainable=True)\n",
    "updates = lasagne.updates.adam(loss, params)\n",
    "train_fn = theano.function([input_var, target_var], loss, updates=updates)\n",
    "test_prediction = lasagne.layers.get_output(l_out, deterministic=True).flatten()\n",
    "test_V = theano.function([input_var], T.gt(test_prediction, 0.))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from random import random, randint\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def player_NN(game):\n",
    "    moves, boards = game._np_branch()\n",
    "    return moves[np.argmax(V(boards))]\n",
    "\n",
    "def player_random(game):\n",
    "    while 1:\n",
    "        r = randint(0,6)\n",
    "        if game.move(r, test=True) is not None:\n",
    "            return r\n",
    "\n",
    "def get_player_MC(N=100):\n",
    "    def player(game):\n",
    "        return MC_agent(game, N=N)\n",
    "    return player\n",
    "\n",
    "def get_player_mixed(*settings):\n",
    "    def player(game):\n",
    "        r = random()\n",
    "        for player, prop in settings:\n",
    "            r-=prop\n",
    "            if r<=0:\n",
    "                return player(game)\n",
    "        return player_random(game)\n",
    "    return player\n",
    "        \n",
    "\n",
    "def vs(player1, player2, display=False):\n",
    "    game = Connect4()\n",
    "    history = []\n",
    "    while game.turn < 42 and not game.is_end():\n",
    "        if game.turn%2 == 0:\n",
    "            m = player1(game)\n",
    "        else:\n",
    "            m = player2(game)\n",
    "        game.move(m)\n",
    "        history.append(m)\n",
    "        if display == 'all':\n",
    "            game.display()\n",
    "    if display:\n",
    "        game.display()\n",
    "    if game.is_end():\n",
    "        return game.turn, history\n",
    "    return 0, history\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def train_if(results):\n",
    "    def train(r, history, old_histories, γ=0.95):\n",
    "        _r = -1 if r ==0 else (r-1)%2\n",
    "        if  _r not in results:\n",
    "            return\n",
    "        game = Connect4()\n",
    "        board_history = []\n",
    "        for m in history:\n",
    "            game.move(m)\n",
    "            board_history.append(game._np_board())\n",
    "        estimate_V = np.zeros(len(history), dtype=floatX)\n",
    "        if _r==-1:\n",
    "            r = 0.\n",
    "        else:\n",
    "            r = 1.\n",
    "        for i in range(len(history)-1, -1, -1):\n",
    "            estimate_V[i]=r\n",
    "            r *= -γ\n",
    "        #old_histories.append( (board_history, estimate_V) )\n",
    "        data0 = board_history \n",
    "        data1 = estimate_V \n",
    "        loss = train_fn(data0, data1)                \n",
    "    return train\n",
    "\n",
    "def vs_test(player1, player2, old_histories=[], ngames=1000, train=None):\n",
    "    result = [0,0,0]\n",
    "    for i in range(ngames):\n",
    "        r, history = vs(player1, player2)\n",
    "        if r == 0:\n",
    "            result[0]+=1\n",
    "        else:\n",
    "            result[1 + (r-1)%2]+=1\n",
    "        if train is not None:\n",
    "            train(r, history, old_histories)\n",
    "            old_histories= old_histories[-20:]\n",
    "    return result\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train2 = train=train_if([-1,0])\n",
    "train1 = train=train_if([-1,1])\n",
    "train_all = train=train_if([-1,0,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 0.0007650852203369141\n",
      "inital result: mc100 vs nn: [4, 18, 78], nn vs mc100 [2, 79, 19]\n",
      "time: 35.474363803863525\n",
      "time: 95.2370011806488\n",
      "#10000 avgloss=0.203823, rand vs nn: [0, 1, 999], nn vs rand [0, 1000, 0]\n",
      "time: 156.89969849586487\n",
      "#20000 avgloss=0.200742, rand vs nn: [0, 0, 1000], nn vs rand [0, 1000, 0]\n",
      "time: 218.47910284996033\n",
      "#30000 avgloss=0.200649, rand vs nn: [0, 2, 998], nn vs rand [0, 1000, 0]\n",
      "time: 287.0495595932007\n",
      "#40000 avgloss=0.202228, rand vs nn: [0, 1, 999], nn vs rand [0, 998, 2]\n",
      "time: 368.0585289001465\n",
      "#50000 avgloss=0.201083, rand vs nn: [0, 8, 992], nn vs rand [0, 999, 1]\n",
      "time: 449.9715254306793\n",
      "#60000 avgloss=0.204487, rand vs nn: [0, 1, 999], nn vs rand [0, 999, 1]\n",
      "time: 530.8367681503296\n",
      "#70000 avgloss=0.199540, rand vs nn: [0, 4, 996], nn vs rand [0, 1000, 0]\n",
      "time: 608.3097262382507\n",
      "#80000 avgloss=0.203173, rand vs nn: [0, 1, 999], nn vs rand [0, 999, 1]\n",
      "time: 686.4618473052979\n",
      "#90000 avgloss=0.202078, rand vs nn: [0, 13, 987], nn vs rand [0, 999, 1]\n",
      "time: 768.4679663181305\n",
      "#100000 midterm avgloss=0.211806\n",
      "            mc100 vs nn: [3, 11, 86], nn vs mc100 [2, 88, 10]\n",
      "time: 895.21546459198\n",
      "#110000 avgloss=0.205254, rand vs nn: [0, 0, 1000], nn vs rand [0, 999, 1]\n",
      "time: 972.3509213924408\n",
      "#120000 avgloss=0.205577, rand vs nn: [0, 8, 992], nn vs rand [0, 1000, 0]\n",
      "time: 1049.383892774582\n",
      "#130000 avgloss=0.209818, rand vs nn: [0, 2, 998], nn vs rand [0, 1000, 0]\n",
      "time: 1130.9517443180084\n",
      "#140000 avgloss=0.199103, rand vs nn: [0, 1, 999], nn vs rand [0, 1000, 0]\n",
      "time: 1208.4004917144775\n",
      "#150000 avgloss=0.212305, rand vs nn: [0, 3, 997], nn vs rand [0, 1000, 0]\n",
      "time: 1285.5828700065613\n",
      "#160000 avgloss=0.206519, rand vs nn: [0, 5, 995], nn vs rand [0, 999, 1]\n",
      "time: 1362.752614736557\n",
      "#170000 avgloss=0.206136, rand vs nn: [0, 2, 998], nn vs rand [0, 1000, 0]\n",
      "time: 1439.8091297149658\n",
      "#180000 avgloss=0.212298, rand vs nn: [0, 3, 997], nn vs rand [0, 1000, 0]\n",
      "time: 1518.050987482071\n",
      "#190000 avgloss=0.210594, rand vs nn: [0, 2, 998], nn vs rand [0, 998, 2]\n",
      "time: 1598.038346529007\n",
      "#200000 midterm avgloss=0.210438\n",
      "            mc100 vs nn: [4, 16, 80], nn vs mc100 [4, 84, 12]\n",
      "time: 1727.0598378181458\n",
      "#210000 avgloss=0.205556, rand vs nn: [0, 2, 998], nn vs rand [0, 1000, 0]\n",
      "time: 1804.6322438716888\n",
      "#220000 avgloss=0.209002, rand vs nn: [0, 2, 998], nn vs rand [0, 999, 1]\n",
      "time: 1882.4241750240326\n",
      "#230000 avgloss=0.205320, rand vs nn: [0, 0, 1000], nn vs rand [0, 1000, 0]\n",
      "time: 1961.3849041461945\n",
      "#240000 avgloss=0.205169, rand vs nn: [0, 5, 995], nn vs rand [0, 1000, 0]\n",
      "time: 2042.312170267105\n",
      "#250000 avgloss=0.205664, rand vs nn: [0, 3, 997], nn vs rand [0, 1000, 0]\n",
      "time: 2120.7388594150543\n",
      "#260000 avgloss=0.207572, rand vs nn: [0, 4, 996], nn vs rand [0, 999, 1]\n",
      "time: 2197.6171793937683\n",
      "#270000 avgloss=0.202642, rand vs nn: [0, 1, 999], nn vs rand [0, 1000, 0]\n",
      "time: 2273.6342113018036\n",
      "#280000 avgloss=0.207629, rand vs nn: [0, 5, 995], nn vs rand [0, 1000, 0]\n",
      "time: 2350.6650488376617\n",
      "#290000 avgloss=0.213101, rand vs nn: [0, 3, 997], nn vs rand [0, 1000, 0]\n",
      "time: 2432.7371962070465\n",
      "#300000 midterm avgloss=0.206149\n",
      "            mc100 vs nn: [1, 16, 83], nn vs mc100 [5, 78, 17]\n",
      "time: 2560.8490359783173\n",
      "#310000 avgloss=0.208536, rand vs nn: [1, 9, 990], nn vs rand [0, 997, 3]\n",
      "time: 2637.0232923030853\n",
      "#320000 avgloss=0.207152, rand vs nn: [0, 3, 997], nn vs rand [0, 999, 1]\n",
      "time: 2714.6850261688232\n",
      "#330000 avgloss=0.208062, rand vs nn: [0, 2, 998], nn vs rand [0, 998, 2]\n",
      "time: 2795.4956352710724\n",
      "#340000 avgloss=0.208879, rand vs nn: [0, 9, 991], nn vs rand [0, 1000, 0]\n",
      "time: 2871.969262599945\n",
      "#350000 avgloss=0.205005, rand vs nn: [0, 2, 998], nn vs rand [0, 1000, 0]\n",
      "time: 2950.0742526054382\n",
      "#360000 avgloss=0.203136, rand vs nn: [0, 3, 997], nn vs rand [0, 999, 1]\n",
      "time: 3027.7502224445343\n",
      "#370000 avgloss=0.200314, rand vs nn: [0, 20, 980], nn vs rand [0, 1000, 0]\n",
      "time: 3109.015181541443\n",
      "#380000 avgloss=0.203747, rand vs nn: [0, 0, 1000], nn vs rand [0, 1000, 0]\n",
      "time: 3188.0484261512756\n",
      "#390000 avgloss=0.204855, rand vs nn: [0, 7, 993], nn vs rand [0, 1000, 0]\n",
      "time: 3267.593944311142\n",
      "#400000 midterm avgloss=0.201647\n",
      "            mc100 vs nn: [1, 11, 88], nn vs mc100 [5, 67, 28]\n",
      "time: 3391.8413395881653\n",
      "#410000 avgloss=0.210115, rand vs nn: [0, 0, 1000], nn vs rand [0, 1000, 0]\n",
      "time: 3469.401000738144\n",
      "#420000 avgloss=0.203219, rand vs nn: [0, 7, 993], nn vs rand [0, 1000, 0]\n",
      "time: 3546.7363245487213\n",
      "#430000 avgloss=0.206649, rand vs nn: [0, 5, 995], nn vs rand [0, 1000, 0]\n",
      "time: 3626.8510389328003\n",
      "#440000 avgloss=0.207680, rand vs nn: [0, 6, 994], nn vs rand [0, 1000, 0]\n",
      "time: 3705.535489797592\n",
      "#450000 avgloss=0.210957, rand vs nn: [0, 0, 1000], nn vs rand [0, 999, 1]\n",
      "time: 3783.3765983581543\n",
      "#460000 avgloss=0.200859, rand vs nn: [0, 6, 994], nn vs rand [0, 1000, 0]\n",
      "time: 3860.466323852539\n",
      "#470000 avgloss=0.201342, rand vs nn: [0, 3, 997], nn vs rand [0, 1000, 0]\n",
      "time: 3939.3514502048492\n",
      "#480000 avgloss=0.209093, rand vs nn: [0, 2, 998], nn vs rand [0, 997, 3]\n",
      "time: 4020.743591070175\n",
      "#490000 avgloss=0.206559, rand vs nn: [0, 1, 999], nn vs rand [0, 999, 1]\n",
      "time: 4098.057268381119\n",
      "#500000 midterm avgloss=0.206939\n",
      "            mc100 vs nn: [5, 24, 71], nn vs mc100 [9, 75, 16]\n",
      "time: 4224.518556594849\n",
      "#510000 avgloss=0.211785, rand vs nn: [0, 6, 994], nn vs rand [0, 1000, 0]\n",
      "time: 4302.80196762085\n",
      "#520000 avgloss=0.205579, rand vs nn: [0, 2, 998], nn vs rand [0, 998, 2]\n",
      "time: 4382.417955160141\n",
      "#530000 avgloss=0.207315, rand vs nn: [0, 2, 998], nn vs rand [0, 1000, 0]\n",
      "time: 4461.675926923752\n",
      "#540000 avgloss=0.202720, rand vs nn: [0, 1, 999], nn vs rand [0, 1000, 0]\n",
      "time: 4540.2880256175995\n",
      "#550000 avgloss=0.205034, rand vs nn: [0, 4, 996], nn vs rand [0, 1000, 0]\n",
      "time: 4618.158355474472\n",
      "#560000 avgloss=0.204187, rand vs nn: [0, 3, 997], nn vs rand [0, 999, 1]\n",
      "time: 4698.289057254791\n",
      "#570000 avgloss=0.198128, rand vs nn: [0, 6, 994], nn vs rand [0, 1000, 0]\n",
      "time: 4777.349367856979\n",
      "#580000 avgloss=0.199781, rand vs nn: [0, 1, 999], nn vs rand [0, 1000, 0]\n",
      "time: 4855.988700866699\n",
      "#590000 avgloss=0.205106, rand vs nn: [0, 4, 996], nn vs rand [0, 998, 2]\n",
      "time: 4935.92052936554\n",
      "#600000 midterm avgloss=0.199197\n",
      "            mc100 vs nn: [0, 20, 80], nn vs mc100 [1, 78, 21]\n",
      "time: 5061.233084201813\n",
      "#610000 avgloss=0.198824, rand vs nn: [0, 2, 998], nn vs rand [0, 1000, 0]\n",
      "time: 5139.966150522232\n",
      "#620000 avgloss=0.200311, rand vs nn: [0, 0, 1000], nn vs rand [0, 998, 2]\n",
      "time: 5218.578981161118\n",
      "#630000 avgloss=0.205611, rand vs nn: [0, 2, 998], nn vs rand [0, 1000, 0]\n",
      "time: 5298.206200361252\n",
      "#640000 avgloss=0.202554, rand vs nn: [0, 0, 1000], nn vs rand [0, 999, 1]\n",
      "time: 5377.469415664673\n",
      "#650000 avgloss=0.207023, rand vs nn: [0, 5, 995], nn vs rand [0, 1000, 0]\n",
      "time: 5455.703722238541\n",
      "#660000 avgloss=0.206085, rand vs nn: [0, 2, 998], nn vs rand [0, 1000, 0]\n",
      "time: 5533.38151216507\n",
      "#670000 avgloss=0.210830, rand vs nn: [0, 0, 1000], nn vs rand [0, 1000, 0]\n",
      "time: 5610.98021030426\n",
      "#680000 avgloss=0.211096, rand vs nn: [0, 2, 998], nn vs rand [0, 1000, 0]\n",
      "time: 5689.753880977631\n",
      "#690000 avgloss=0.205594, rand vs nn: [0, 3, 997], nn vs rand [0, 1000, 0]\n",
      "time: 5768.301014661789\n",
      "#700000 midterm avgloss=0.199961\n",
      "            mc100 vs nn: [1, 9, 90], nn vs mc100 [1, 84, 15]\n",
      "time: 5892.625888586044\n",
      "#710000 avgloss=0.199076, rand vs nn: [0, 5, 995], nn vs rand [0, 1000, 0]\n",
      "time: 5971.148954153061\n",
      "#720000 avgloss=0.202677, rand vs nn: [0, 3, 997], nn vs rand [0, 1000, 0]\n",
      "time: 6048.684808731079\n",
      "#730000 avgloss=0.198285, rand vs nn: [0, 19, 981], nn vs rand [0, 1000, 0]\n",
      "time: 6128.314256668091\n",
      "#740000 avgloss=0.201696, rand vs nn: [0, 2, 998], nn vs rand [0, 999, 1]\n",
      "time: 6208.047273159027\n",
      "#750000 avgloss=0.195916, rand vs nn: [0, 5, 995], nn vs rand [0, 1000, 0]\n",
      "time: 6286.639931678772\n",
      "#760000 avgloss=0.198392, rand vs nn: [0, 2, 998], nn vs rand [0, 1000, 0]\n",
      "time: 6364.603930473328\n",
      "#770000 avgloss=0.198728, rand vs nn: [0, 2, 998], nn vs rand [0, 1000, 0]\n",
      "time: 6445.268532752991\n",
      "#780000 avgloss=0.201662, rand vs nn: [0, 7, 993], nn vs rand [0, 1000, 0]\n",
      "time: 6522.464213609695\n",
      "#790000 avgloss=0.203139, rand vs nn: [0, 6, 994], nn vs rand [0, 1000, 0]\n",
      "time: 6601.327981233597\n",
      "#800000 midterm avgloss=0.202436\n",
      "            mc100 vs nn: [1, 13, 86], nn vs mc100 [5, 82, 13]\n",
      "time: 6732.348340034485\n",
      "#810000 avgloss=0.200211, rand vs nn: [0, 4, 996], nn vs rand [0, 1000, 0]\n",
      "time: 6810.087117433548\n",
      "#820000 avgloss=0.201288, rand vs nn: [0, 8, 992], nn vs rand [0, 999, 1]\n",
      "time: 6889.365455627441\n",
      "#830000 avgloss=0.204306, rand vs nn: [0, 2, 998], nn vs rand [0, 1000, 0]\n",
      "time: 6968.315800666809\n",
      "#840000 avgloss=0.203301, rand vs nn: [0, 12, 988], nn vs rand [0, 1000, 0]\n",
      "time: 7046.941210269928\n",
      "#850000 avgloss=0.202535, rand vs nn: [0, 7, 993], nn vs rand [0, 1000, 0]\n",
      "time: 7126.296331644058\n",
      "#860000 avgloss=0.203629, rand vs nn: [0, 8, 992], nn vs rand [0, 1000, 0]\n",
      "time: 7204.815500736237\n",
      "#870000 avgloss=0.202422, rand vs nn: [0, 16, 984], nn vs rand [0, 1000, 0]\n",
      "time: 7283.343268156052\n",
      "#880000 avgloss=0.199340, rand vs nn: [0, 5, 995], nn vs rand [0, 1000, 0]\n",
      "time: 7358.1997101306915\n",
      "#890000 avgloss=0.196979, rand vs nn: [0, 2, 998], nn vs rand [0, 1000, 0]\n",
      "time: 7439.784137248993\n",
      "#900000 midterm avgloss=0.202144\n",
      "            mc100 vs nn: [3, 16, 81], nn vs mc100 [3, 85, 12]\n",
      "time: 7567.539828538895\n",
      "#910000 avgloss=0.199944, rand vs nn: [0, 3, 997], nn vs rand [0, 1000, 0]\n",
      "time: 7644.782443523407\n",
      "#920000 avgloss=0.196523, rand vs nn: [0, 10, 990], nn vs rand [0, 1000, 0]\n",
      "time: 7724.228224992752\n",
      "#930000 avgloss=0.195388, rand vs nn: [0, 4, 996], nn vs rand [0, 1000, 0]\n",
      "time: 7804.287939071655\n",
      "#940000 avgloss=0.195078, rand vs nn: [0, 5, 995], nn vs rand [0, 1000, 0]\n",
      "time: 7880.569224357605\n",
      "#950000 avgloss=0.196193, rand vs nn: [0, 0, 1000], nn vs rand [0, 1000, 0]\n",
      "time: 7958.7024874687195\n",
      "#960000 avgloss=0.203324, rand vs nn: [0, 7, 993], nn vs rand [0, 999, 1]\n",
      "time: 8037.038475751877\n",
      "#970000 avgloss=0.203978, rand vs nn: [0, 4, 996], nn vs rand [0, 1000, 0]\n",
      "time: 8112.5142776966095\n",
      "#980000 avgloss=0.206063, rand vs nn: [0, 1, 999], nn vs rand [0, 1000, 0]\n",
      "time: 8189.433959722519\n",
      "#990000 avgloss=0.208722, rand vs nn: [0, 0, 1000], nn vs rand [0, 1000, 0]\n",
      "time: 8266.999913454056\n",
      "#1000000 midterm avgloss=0.208114\n",
      "            mc100 vs nn: [3, 15, 82], nn vs mc100 [5, 86, 9]\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import sys\n",
    "start_time=time.time()\n",
    "def run_game(V, verbose = False, ɛ=0.1, γ=0.95):\n",
    "    game = Connect4()\n",
    "    history=[]\n",
    "    runtime_V=[]\n",
    "    while game.turn < 42 and not game.is_end():\n",
    "        s = 1 if game.turn%2 == 0 else -1\n",
    "        moves, boards = game._np_branch()\n",
    "        #print(boards)\n",
    "        if random() < ɛ:\n",
    "            idx = randint(0, len(moves)-1)\n",
    "            values=[None]*len(moves)\n",
    "        else:\n",
    "            values = V(boards)\n",
    "            idx = np.argmax(values)\n",
    "        m = moves[idx]\n",
    "        game.move(m)\n",
    "        history.append(boards[idx])\n",
    "        runtime_V.append(values[idx])\n",
    "    if game.is_end():\n",
    "        result = 1.\n",
    "    else:\n",
    "        result = 0. # Tie\n",
    "    # train here\n",
    "    #game.display()\n",
    "    estimate_V = np.zeros(len(history), dtype=floatX)\n",
    "    r = result\n",
    "    for i in range(len(history)-1, -1, -1):\n",
    "        estimate_V[i]=r\n",
    "        r *= -γ\n",
    "    loss = train_fn(np.array(history, dtype=floatX), estimate_V)\n",
    "    return loss\n",
    "total_loss = 0\n",
    "N = 10000\n",
    "print(\"time:\",time.time()-start_time)\n",
    "print(\"inital result: mc100 vs nn: %s, nn vs mc100 %s\"%(vs_test(get_player_MC(100), player_NN, ngames=100), \n",
    "                                                          vs_test(player_NN, get_player_MC(100), ngames=100)))\n",
    "print(\"time:\",time.time()-start_time)\n",
    "for i in range(100*N):\n",
    "    total_loss += run_game(V)\n",
    "    if i%N==N-1:\n",
    "        print(\"time:\",time.time()-start_time)\n",
    "        if i%(10*N)==10*N-1:\n",
    "            print(\"#%d midterm avgloss=%f\"%(i+1, total_loss/N))\n",
    "            print(\"            mc100 vs nn: %s, nn vs mc100 %s\"%(vs_test(get_player_MC(100), player_NN, ngames=100, train=train2), \n",
    "                                                          vs_test(player_NN, get_player_MC(100), ngames=100, train=train1)) )\n",
    "        else:\n",
    "            print(\"#%d avgloss=%f, rand vs nn: %s, nn vs rand %s\"%(i+1, total_loss/N, vs_test(player_random, player_NN, ngames=1000, train=train2), \n",
    "                                                              vs_test(player_NN, player_random, ngames=1000, train=train1)) )\n",
    "        total_loss = 0\n",
    "        sys.stdout.flush()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "values = lasagne.layers.get_all_param_values(l_out)\n",
    "pickle.dump(values,open(\"c4-simple2.pkl\",\"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load\n",
    "import pickle\n",
    "values = pickle.load(open(\"c4-simple2.pkl\",\"rb\"))\n",
    "lasagne.layers.set_all_param_values(l_out, values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"width: 200px; height:180px;position: relative;background: blue\"><img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 140px; left: 7px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 113px; left: 7px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 87px; left: 7px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 60px; left: 7px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 33px; left: 7px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 7px; left: 7px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 140px; left: 33px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 113px; left: 33px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 87px; left: 33px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 60px; left: 33px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 33px; left: 33px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 7px; left: 33px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 140px; left: 60px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 113px; left: 60px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 87px; left: 60px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 60px; left: 60px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 33px; left: 60px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 7px; left: 60px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 140px; left: 87px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 113px; left: 87px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 87px; left: 87px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 60px; left: 87px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 33px; left: 87px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 7px; left: 87px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 140px; left: 113px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 113px; left: 113px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 87px; left: 113px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 60px; left: 113px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 33px; left: 113px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 7px; left: 113px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 140px; left: 140px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 113px; left: 140px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 87px; left: 140px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 60px; left: 140px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 33px; left: 140px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 7px; left: 140px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 140px; left: 167px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 113px; left: 167px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 87px; left: 167px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 60px; left: 167px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 33px; left: 167px;margin-top: 0;z-index: 0' />\n",
       "<img src='img/empty.png' width='23px' height='23px' style='position: absolute; top: 7px; left: 167px;margin-top: 0;z-index: 0' /><img src='img/red_coin.png' width='23px' height='23px' style='position: absolute; top: 140px; left: 7px;margin-top: 0;z-index: 2' />\n",
       "<img src='img/red_coin.png' width='23px' height='23px' style='position: absolute; top: 113px; left: 7px;margin-top: 0;z-index: 2' />\n",
       "<img src='img/red_coin.png' width='23px' height='23px' style='position: absolute; top: 87px; left: 7px;margin-top: 0;z-index: 2' />\n",
       "<img src='img/yellow_coin.png' width='23px' height='23px' style='position: absolute; top: 60px; left: 7px;margin-top: 0;z-index: 2' />\n",
       "<img src='img/red_coin.png' width='23px' height='23px' style='position: absolute; top: 140px; left: 33px;margin-top: 0;z-index: 2' />\n",
       "<img src='img/red_coin.png' width='23px' height='23px' style='position: absolute; top: 113px; left: 33px;margin-top: 0;z-index: 2' />\n",
       "<img src='img/red_coin.png' width='23px' height='23px' style='position: absolute; top: 87px; left: 33px;margin-top: 0;z-index: 2' />\n",
       "<img src='img/yellow_coin.png' width='23px' height='23px' style='position: absolute; top: 60px; left: 33px;margin-top: 0;z-index: 2' />\n",
       "<img src='img/yellow_coin.png' width='23px' height='23px' style='position: absolute; top: 33px; left: 33px;margin-top: 0;z-index: 2' />\n",
       "<img src='img/red_coin.png' width='23px' height='23px' style='position: absolute; top: 7px; left: 33px;margin-top: 0;z-index: 2' />\n",
       "<img src='img/yellow_coin.png' width='23px' height='23px' style='position: absolute; top: 140px; left: 60px;margin-top: 0;z-index: 2' />\n",
       "<img src='img/red_coin.png' width='23px' height='23px' style='position: absolute; top: 113px; left: 60px;margin-top: 0;z-index: 2' />\n",
       "<img src='img/yellow_coin.png' width='23px' height='23px' style='position: absolute; top: 140px; left: 87px;margin-top: 0;z-index: 2' />\n",
       "<img src='img/red_coin.png' width='23px' height='23px' style='position: absolute; top: 113px; left: 87px;margin-top: 0;z-index: 2' />\n",
       "<img src='img/yellow_coin.png' width='23px' height='23px' style='position: absolute; top: 87px; left: 87px;margin-top: 0;z-index: 2' />\n",
       "<img src='img/yellow_coin.png' width='23px' height='23px' style='position: absolute; top: 140px; left: 113px;margin-top: 0;z-index: 2' />\n",
       "<img src='img/yellow_coin.png' width='23px' height='23px' style='position: absolute; top: 113px; left: 113px;margin-top: 0;z-index: 2' />\n",
       "<img src='img/yellow_coin.png' width='23px' height='23px' style='position: absolute; top: 87px; left: 113px;margin-top: 0;z-index: 2' />\n",
       "<img src='img/red_coin.png' width='23px' height='23px' style='position: absolute; top: 60px; left: 113px;margin-top: 0;z-index: 2' />\n",
       "<img src='img/red_coin.png' width='23px' height='23px' style='position: absolute; top: 140px; left: 140px;margin-top: 0;z-index: 2' />\n",
       "<img src='img/yellow_coin.png' width='23px' height='23px' style='position: absolute; top: 113px; left: 140px;margin-top: 0;z-index: 2' /></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.display import clear_output\n",
    "game = Connect4()\n",
    "c = randint(0, 1)\n",
    "while 1:\n",
    "    clear_output()\n",
    "    display(HTML(game._repr_html_()))\n",
    "    if game.turn >= 42 or game.is_end():\n",
    "        break\n",
    "    if game.turn%2 == c:\n",
    "        #game.move(MC_agent(game))\n",
    "        #while game.move(randint(0,6)) is None: continue\n",
    "        moves, boards = game._np_branch()\n",
    "        game.move(moves[np.argmax(test_V(boards))])\n",
    "    else:\n",
    "        game.move(int(input(\"your turn\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}