Playing_tic_tac_toe_with_muzero_algorithm.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "请点击[此处](https://ai.baidu.com/docs#/AIStudio_Project_Notebook/a38e5576)查看本环境基本用法.  <br>\n",
    "Please click [here ](https://ai.baidu.com/docs#/AIStudio_Project_Notebook/a38e5576) for more detailed instructions. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "# 构建简易Tic-Tac-Toe环境\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   1 2 3\n",
      "A _ _ _\n",
      "B O _ _\n",
      "C _ _ _\n",
      "record = B1\n",
      "input feature\n",
      "[[[0. 0. 0.]\n",
      "  [0. 0. 0.]\n",
      "  [0. 0. 0.]]\n",
      "\n",
      " [[0. 0. 0.]\n",
      "  [1. 0. 0.]\n",
      "  [0. 0. 0.]]]\n",
      "input feature\n",
      "[[[1. 0. 0.]\n",
      "  [0. 0. 0.]\n",
      "  [0. 0. 0.]]\n",
      "\n",
      " [[0. 0. 0.]\n",
      "  [0. 1. 0.]\n",
      "  [0. 1. 0.]]]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\r\n",
    "\r\n",
    "BLACK, WHITE = 1, -1  # first turn or second turn player\r\n",
    "\r\n",
    "class State:\r\n",
    "    '''Board implementation of Tic-Tac-Toe'''\r\n",
    "    X, Y = 'ABC',  '123'\r\n",
    "    C = {0: '_', BLACK: 'O', WHITE: 'X'}\r\n",
    "\r\n",
    "    def __init__(self):\r\n",
    "        self.board = np.zeros((3, 3)) # (x, y)\r\n",
    "        self.color = 1\r\n",
    "        self.win_color = 0\r\n",
    "        self.record = []\r\n",
    "\r\n",
    "    def action2str(self, a):\r\n",
    "        return self.X[a // 3] + self.Y[a % 3]\r\n",
    "\r\n",
    "    def str2action(self, s):\r\n",
    "        return self.X.find(s[0]) * 3 + self.Y.find(s[1])\r\n",
    "\r\n",
    "    def record_string(self):\r\n",
    "        return ' '.join([self.action2str(a) for a in self.record])\r\n",
    "\r\n",
    "    def __str__(self):\r\n",
    "        # output board.\r\n",
    "        s = '   ' + ' '.join(self.Y) + '\\n'\r\n",
    "        for i in range(3):\r\n",
    "            s += self.X[i] + ' ' + ' '.join([self.C[self.board[i, j]] for j in range(3)]) + '\\n'\r\n",
    "        s += 'record = ' + self.record_string()\r\n",
    "        return s\r\n",
    "\r\n",
    "    def play(self, action):\r\n",
    "        # state transition function\r\n",
    "        # action is position inerger (0~8) or string representation of action sequence\r\n",
    "        if isinstance(action, str):\r\n",
    "            for astr in action.split():\r\n",
    "                self.play(self.str2action(astr))\r\n",
    "            return self\r\n",
    "\r\n",
    "        x, y = action // 3, action % 3\r\n",
    "        self.board[x, y] = self.color\r\n",
    "\r\n",
    "        # check whether 3 stones are on the line\r\n",
    "        if self.board[x, :].sum() == 3 * self.color \\\r\n",
    "          or self.board[:, y].sum() == 3 * self.color \\\r\n",
    "          or (x == y and np.diag(self.board, k=0).sum() == 3 * self.color) \\\r\n",
    "          or (x == 2 - y and np.diag(self.board[::-1,:], k=0).sum() == 3 * self.color):\r\n",
    "            self.win_color = self.color\r\n",
    "\r\n",
    "        self.color = -self.color\r\n",
    "        self.record.append(action)\r\n",
    "        return self\r\n",
    "\r\n",
    "    def terminal(self):\r\n",
    "        # terminal state check\r\n",
    "        return self.win_color != 0 or len(self.record) == 3 * 3\r\n",
    "\r\n",
    "    def terminal_reward(self):\r\n",
    "        # terminal reward \r\n",
    "        return self.win_color if self.color == BLACK else -self.win_color\r\n",
    "\r\n",
    "    def legal_actions(self):\r\n",
    "        # list of legal actions on each state\r\n",
    "        return [a for a in range(3 * 3) if self.board[a // 3, a % 3] == 0]\r\n",
    "\r\n",
    "    def feature(self):\r\n",
    "        # input tensor for neural net (state)\r\n",
    "        return np.stack([self.board == self.color, self.board == -self.color]).astype(np.float32)\r\n",
    "\r\n",
    "    def action_feature(self, action):\r\n",
    "        # input tensor for neural net (action)\r\n",
    "        a = np.zeros((1, 3, 3), dtype=np.float32)\r\n",
    "        a[0, action // 3, action % 3] = 1\r\n",
    "        return a\r\n",
    "\r\n",
    "state = State().play('B1')\r\n",
    "print(state)\r\n",
    "print('input feature')\r\n",
    "print(state.feature())\r\n",
    "state = State().play('B2 A1 C2')\r\n",
    "print('input feature')\r\n",
    "print(state.feature())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "# 构建神经网络"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import paddle\r\n",
    "import paddle.nn as nn\r\n",
    "import paddle.nn.functional as F\r\n",
    "\r\n",
    "class Conv(paddle.nn.Layer):\r\n",
    "    def __init__(self, filters0, filters1, kernel_size, bn=False):\r\n",
    "        super().__init__()\r\n",
    "        self.conv = nn.Conv2D(filters0, filters1, kernel_size, stride=1, padding=kernel_size//2, bias_attr=False)\r\n",
    "        self.bn = None\r\n",
    "        if bn:\r\n",
    "            self.bn = nn.BatchNorm2D(filters1)\r\n",
    "\r\n",
    "    def forward(self, x):\r\n",
    "        h = self.conv(x)\r\n",
    "        if self.bn is not None:\r\n",
    "            h = self.bn(h)\r\n",
    "        return h\r\n",
    "\r\n",
    "class ResidualBlock(paddle.nn.Layer):\r\n",
    "    def __init__(self, filters):\r\n",
    "        super().__init__()\r\n",
    "        self.conv = Conv(filters, filters, 3, True)\r\n",
    "\r\n",
    "    def forward(self, x):\r\n",
    "        return F.relu(x + (self.conv(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "num_filters = 16\r\n",
    "num_blocks = 4\r\n",
    "\r\n",
    "class Representation(paddle.nn.Layer):\r\n",
    "    ''' Conversion from observation to inner abstract state '''\r\n",
    "    def __init__(self, input_shape):\r\n",
    "        super().__init__()\r\n",
    "        self.input_shape = input_shape\r\n",
    "        self.board_size = self.input_shape[1] * self.input_shape[2]\r\n",
    "\r\n",
    "        self.layer0 = Conv(self.input_shape[0], num_filters, 3, bn=True)\r\n",
    "        self.blocks = nn.LayerList([ResidualBlock(num_filters) for _ in range(num_blocks)])\r\n",
    "\r\n",
    "    def forward(self, x):\r\n",
    "        h = F.relu(self.layer0(x))\r\n",
    "        for block in self.blocks:\r\n",
    "            h = block(h)\r\n",
    "        return h\r\n",
    "\r\n",
    "    def inference(self, x):\r\n",
    "        self.eval()\r\n",
    "        with paddle.no_grad(): \r\n",
    "            rp = self.forward(paddle.to_tensor(x).unsqueeze(0))\r\n",
    "        return rp.cpu().numpy()[0]\r\n",
    "\r\n",
    "class Prediction(paddle.nn.Layer):\r\n",
    "    ''' Policy and value prediction from inner abstract state '''\r\n",
    "    def __init__(self, action_shape):\r\n",
    "        super().__init__()\r\n",
    "        self.board_size = np.prod(action_shape[1:])\r\n",
    "        self.action_size = action_shape[0] * self.board_size\r\n",
    "\r\n",
    "        self.conv_p1 = Conv(num_filters, 4, 1, bn=True)\r\n",
    "        self.conv_p2 = Conv(4, 1, 1)\r\n",
    "\r\n",
    "        self.conv_v = Conv(num_filters, 4, 1, bn=True)\r\n",
    "        self.fc_v = nn.Linear(self.board_size * 4, 1, bias_attr=False)\r\n",
    "\r\n",
    "    def forward(self, rp):\r\n",
    "        h_p = F.relu(self.conv_p1(rp))\r\n",
    "        h_p = self.conv_p2(h_p).reshape([-1, self.action_size])\r\n",
    "\r\n",
    "        h_v = F.relu(self.conv_v(rp))\r\n",
    "        h_v = self.fc_v(h_v.reshape([-1, self.board_size * 4]))\r\n",
    "\r\n",
    "        # range of value is -1 ~ 1\r\n",
    "        return F.softmax(h_p, axis=-1), paddle.tanh(h_v)\r\n",
    "\r\n",
    "    def inference(self, rp):\r\n",
    "        self.eval()\r\n",
    "        with paddle.no_grad():\r\n",
    "            \r\n",
    "            p, v = self.forward(paddle.to_tensor(rp).unsqueeze(0))\r\n",
    "        return p.cpu().numpy()[0], v.cpu().numpy()[0][0]\r\n",
    "\r\n",
    "class Dynamics(paddle.nn.Layer):\r\n",
    "    '''Abstract state transition'''\r\n",
    "    def __init__(self, rp_shape, act_shape):\r\n",
    "        super().__init__()\r\n",
    "        self.rp_shape = rp_shape\r\n",
    "        self.layer0 = Conv(rp_shape[0] + act_shape[0], num_filters, 3, bn=True)\r\n",
    "        self.blocks = nn.LayerList([ResidualBlock(num_filters) for _ in range(num_blocks)])\r\n",
    "\r\n",
    "    def forward(self, rp, a):\r\n",
    "        h = paddle.concat([rp, a], axis=1)\r\n",
    "        h = self.layer0(h)\r\n",
    "        for block in self.blocks:\r\n",
    "            h = block(h)\r\n",
    "        return h\r\n",
    "\r\n",
    "    def inference(self, rp, a):\r\n",
    "        self.eval()\r\n",
    "        with paddle.no_grad():\r\n",
    "            rp = self.forward(paddle.to_tensor(rp).unsqueeze(0), paddle.to_tensor(a).unsqueeze(0))\r\n",
    "        return rp.cpu().numpy()[0]\r\n",
    "\r\n",
    "class Net(paddle.nn.Layer):\r\n",
    "    '''Whole net'''\r\n",
    "    def __init__(self):\r\n",
    "        super().__init__()\r\n",
    "        state = State()\r\n",
    "        input_shape = state.feature().shape\r\n",
    "        action_shape = state.action_feature(0).shape\r\n",
    "        rp_shape = (num_filters, *input_shape[1:])\r\n",
    "\r\n",
    "        self.representation = Representation(input_shape)\r\n",
    "        self.prediction = Prediction(action_shape)\r\n",
    "        self.dynamics = Dynamics(rp_shape, action_shape)  \r\n",
    "    \r\n",
    "    def forward(self):\r\n",
    "        pass\r\n",
    "\r\n",
    "    def predict(self, state0, path):\r\n",
    "        '''Predict p and v from original state and path'''\r\n",
    "        outputs = []\r\n",
    "        x = state0.feature()\r\n",
    "        rp = self.representation.inference(x)\r\n",
    "        outputs.append(self.prediction.inference(rp))\r\n",
    "        for action in path:\r\n",
    "            a = state0.action_feature(action)\r\n",
    "            rp = self.dynamics.inference(rp, a)\r\n",
    "            outputs.append(self.prediction.inference(rp))\r\n",
    "        return outputs\r\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   1 2 3\n",
      "A _ _ _\n",
      "B _ _ _\n",
      "C _ _ _\n",
      "record = \n",
      "[0.11111111 0.11111111 0.11111111 0.11111111 0.11111111 0.11111111\n",
      " 0.11111111 0.11111111 0.11111111] 0.0\n",
      "p = \n",
      "[[[111 111 111]\n",
      "  [111 111 111]\n",
      "  [111 111 111]]]\n",
      "v =  0.0\n",
      "\n",
      "----------------------------------------------------------------------------\n",
      "  Layer (type)       Input Shape          Output Shape         Param #    \n",
      "============================================================================\n",
      "   Conv2D-131       [[1, 1, 3, 3]]       [1, 16, 3, 3]           144      \n",
      "BatchNorm2D-121    [[1, 16, 3, 3]]       [1, 16, 3, 3]           64       \n",
      "    Conv-131        [[1, 1, 3, 3]]       [1, 16, 3, 3]            0       \n",
      "   Conv2D-132      [[1, 16, 3, 3]]       [1, 16, 3, 3]          2,304     \n",
      "BatchNorm2D-122    [[1, 16, 3, 3]]       [1, 16, 3, 3]           64       \n",
      "    Conv-132       [[1, 16, 3, 3]]       [1, 16, 3, 3]            0       \n",
      "ResidualBlock-81   [[1, 16, 3, 3]]       [1, 16, 3, 3]            0       \n",
      "   Conv2D-133      [[1, 16, 3, 3]]       [1, 16, 3, 3]          2,304     \n",
      "BatchNorm2D-123    [[1, 16, 3, 3]]       [1, 16, 3, 3]           64       \n",
      "    Conv-133       [[1, 16, 3, 3]]       [1, 16, 3, 3]            0       \n",
      "ResidualBlock-82   [[1, 16, 3, 3]]       [1, 16, 3, 3]            0       \n",
      "   Conv2D-134      [[1, 16, 3, 3]]       [1, 16, 3, 3]          2,304     \n",
      "BatchNorm2D-124    [[1, 16, 3, 3]]       [1, 16, 3, 3]           64       \n",
      "    Conv-134       [[1, 16, 3, 3]]       [1, 16, 3, 3]            0       \n",
      "ResidualBlock-83   [[1, 16, 3, 3]]       [1, 16, 3, 3]            0       \n",
      "   Conv2D-135      [[1, 16, 3, 3]]       [1, 16, 3, 3]          2,304     \n",
      "BatchNorm2D-125    [[1, 16, 3, 3]]       [1, 16, 3, 3]           64       \n",
      "    Conv-135       [[1, 16, 3, 3]]       [1, 16, 3, 3]            0       \n",
      "ResidualBlock-84   [[1, 16, 3, 3]]       [1, 16, 3, 3]            0       \n",
      "============================================================================\n",
      "Total params: 9,680\n",
      "Trainable params: 9,360\n",
      "Non-trainable params: 320\n",
      "----------------------------------------------------------------------------\n",
      "Input size (MB): 0.00\n",
      "Forward/backward pass size (MB): 0.02\n",
      "Params size (MB): 0.04\n",
      "Estimated Total Size (MB): 0.06\n",
      "----------------------------------------------------------------------------\n",
      "\n",
      "---------------------------------------------------------------------------\n",
      " Layer (type)       Input Shape          Output Shape         Param #    \n",
      "===========================================================================\n",
      "  Conv2D-136      [[1, 16, 3, 3]]        [1, 4, 3, 3]           64       \n",
      "BatchNorm2D-126    [[1, 4, 3, 3]]        [1, 4, 3, 3]           16       \n",
      "   Conv-136       [[1, 16, 3, 3]]        [1, 4, 3, 3]            0       \n",
      "  Conv2D-137       [[1, 4, 3, 3]]        [1, 1, 3, 3]            4       \n",
      "   Conv-137        [[1, 4, 3, 3]]        [1, 1, 3, 3]            0       \n",
      "  Conv2D-138      [[1, 16, 3, 3]]        [1, 4, 3, 3]           64       \n",
      "BatchNorm2D-127    [[1, 4, 3, 3]]        [1, 4, 3, 3]           16       \n",
      "   Conv-138       [[1, 16, 3, 3]]        [1, 4, 3, 3]            0       \n",
      "   Linear-11         [[1, 36]]              [1, 1]              36       \n",
      "===========================================================================\n",
      "Total params: 200\n",
      "Trainable params: 168\n",
      "Non-trainable params: 32\n",
      "---------------------------------------------------------------------------\n",
      "Input size (MB): 0.00\n",
      "Forward/backward pass size (MB): 0.00\n",
      "Params size (MB): 0.00\n",
      "Estimated Total Size (MB): 0.00\n",
      "---------------------------------------------------------------------------\n",
      "\n",
      "----------------------------------------------------------------------------\n",
      "  Layer (type)       Input Shape          Output Shape         Param #    \n",
      "============================================================================\n",
      "   Conv2D-139      [[3, 17, 3, 3]]       [3, 16, 3, 3]          2,448     \n",
      "BatchNorm2D-128    [[3, 16, 3, 3]]       [3, 16, 3, 3]           64       \n",
      "    Conv-139       [[3, 17, 3, 3]]       [3, 16, 3, 3]            0       \n",
      "   Conv2D-140      [[3, 16, 3, 3]]       [3, 16, 3, 3]          2,304     \n",
      "BatchNorm2D-129    [[3, 16, 3, 3]]       [3, 16, 3, 3]           64       \n",
      "    Conv-140       [[3, 16, 3, 3]]       [3, 16, 3, 3]            0       \n",
      "ResidualBlock-85   [[3, 16, 3, 3]]       [3, 16, 3, 3]            0       \n",
      "   Conv2D-141      [[3, 16, 3, 3]]       [3, 16, 3, 3]          2,304     \n",
      "BatchNorm2D-130    [[3, 16, 3, 3]]       [3, 16, 3, 3]           64       \n",
      "    Conv-141       [[3, 16, 3, 3]]       [3, 16, 3, 3]            0       \n",
      "ResidualBlock-86   [[3, 16, 3, 3]]       [3, 16, 3, 3]            0       \n",
      "   Conv2D-142      [[3, 16, 3, 3]]       [3, 16, 3, 3]          2,304     \n",
      "BatchNorm2D-131    [[3, 16, 3, 3]]       [3, 16, 3, 3]           64       \n",
      "    Conv-142       [[3, 16, 3, 3]]       [3, 16, 3, 3]            0       \n",
      "ResidualBlock-87   [[3, 16, 3, 3]]       [3, 16, 3, 3]            0       \n",
      "   Conv2D-143      [[3, 16, 3, 3]]       [3, 16, 3, 3]          2,304     \n",
      "BatchNorm2D-132    [[3, 16, 3, 3]]       [3, 16, 3, 3]           64       \n",
      "    Conv-143       [[3, 16, 3, 3]]       [3, 16, 3, 3]            0       \n",
      "ResidualBlock-88   [[3, 16, 3, 3]]       [3, 16, 3, 3]            0       \n",
      "============================================================================\n",
      "Total params: 11,984\n",
      "Trainable params: 11,664\n",
      "Non-trainable params: 320\n",
      "----------------------------------------------------------------------------\n",
      "Input size (MB): 0.00\n",
      "Forward/backward pass size (MB): 0.06\n",
      "Params size (MB): 0.05\n",
      "Estimated Total Size (MB): 0.11\n",
      "----------------------------------------------------------------------------\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'total_params': 11984, 'trainable_params': 11664}"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def show_net(net, state):\r\n",
    "    '''Display policy (p) and value (v)'''\r\n",
    "    print(state)\r\n",
    "    p, v = net.predict(state, [])[-1]\r\n",
    "    print(p,v)\r\n",
    "    print('p = ')\r\n",
    "    print((p * 1000).astype(int).reshape((-1, *net.representation.input_shape[1:3])))\r\n",
    "    print('v = ', v)\r\n",
    "    print()\r\n",
    "\r\n",
    "#  Outputs before training\r\n",
    "show_net(Net(), State())\r\n",
    "paddle.summary(Representation([1,3,3]), (1,1,3,3))\r\n",
    "paddle.summary(Prediction((1,3,3)), (1,16,3,3))\r\n",
    "paddle.summary(Dynamics([16,3,3],[1,3,3]), [(3,16,3,3),(3,1,3,3)])\r\n",
    "\r\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "# 基于神经网络的蒙特卡洛搜索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class Node:\r\n",
    "    '''Search result of one abstract (or root) state'''\r\n",
    "    def __init__(self, p, v):\r\n",
    "        self.p, self.v = p, v\r\n",
    "        self.n, self.q_sum = np.zeros_like(p), np.zeros_like(p)\r\n",
    "        self.n_all, self.q_sum_all = 1, v / 2 # prior\r\n",
    "\r\n",
    "    def update(self, action, q_new):\r\n",
    "        # Update\r\n",
    "        self.n[action] += 1\r\n",
    "        self.q_sum[action] += q_new\r\n",
    "\r\n",
    "        # Update overall stats\r\n",
    "        self.n_all += 1\r\n",
    "        self.q_sum_all += q_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import time\r\n",
    "import copy\r\n",
    "\r\n",
    "class Tree:\r\n",
    "    '''Monte Carlo Tree'''\r\n",
    "    def __init__(self, net):\r\n",
    "        self.net = net\r\n",
    "        self.nodes = {}\r\n",
    "\r\n",
    "    def search(self, state, path, rp, depth):\r\n",
    "        # Return predicted value from new state\r\n",
    "        key = state.record_string()\r\n",
    "        if len(path) > 0:\r\n",
    "            key += '|' + ' '.join(map(state.action2str, path))\r\n",
    "        if key not in self.nodes:\r\n",
    "            p, v = self.net.prediction.inference(rp)\r\n",
    "            self.nodes[key] = Node(p, v)\r\n",
    "            return v\r\n",
    "\r\n",
    "        # State transition by an action selected from bandit\r\n",
    "        node = self.nodes[key]\r\n",
    "        p = node.p\r\n",
    "        mask = np.zeros_like(p)\r\n",
    "        if depth == 0:\r\n",
    "            # Add noise to policy on the root node\r\n",
    "            p = 0.75 * p + 0.25 * np.random.dirichlet([0.15] * len(p))\r\n",
    "            # On the root node, we choose action only from legal actions\r\n",
    "            mask[state.legal_actions()] = 1\r\n",
    "            p *= mask\r\n",
    "            p /= p.sum() + 1e-16\r\n",
    "\r\n",
    "        n, q_sum = 1 + node.n, node.q_sum_all / node.n_all + node.q_sum\r\n",
    "        ucb = q_sum / n + 2.0 * np.sqrt(node.n_all) * p / n + mask * 4 # PUCB formula\r\n",
    "        best_action = np.argmax(ucb)\r\n",
    "\r\n",
    "        # Search next state by recursively calling this function\r\n",
    "        rp_next = self.net.dynamics.inference(rp, state.action_feature(best_action))\r\n",
    "        path.append(best_action)\r\n",
    "        q_new = -self.search(state, path, rp_next, depth + 1) # With the assumption of changing player by turn\r\n",
    "        node.update(best_action, q_new)\r\n",
    "\r\n",
    "        return q_new\r\n",
    "\r\n",
    "    def think(self, state, num_simulations, temperature = 0, show=False):\r\n",
    "        # End point of MCTS\r\n",
    "        if show:\r\n",
    "            print(state)\r\n",
    "        start, prev_time = time.time(), 0\r\n",
    "        for _ in range(num_simulations):\r\n",
    "            self.search(state, [], self.net.representation.inference(state.feature()), depth=0)\r\n",
    "\r\n",
    "            # Display search result on every second\r\n",
    "            if show:\r\n",
    "                tmp_time = time.time() - start\r\n",
    "                if int(tmp_time) > int(prev_time):\r\n",
    "                    prev_time = tmp_time\r\n",
    "                    root, pv = self.nodes[state.record_string()], self.pv(state)\r\n",
    "                    print('%.2f sec. best %s. q = %.4f. n = %d / %d. pv = %s'\r\n",
    "                          % (tmp_time, state.action2str(pv[0]), root.q_sum[pv[0]] / root.n[pv[0]],\r\n",
    "                             root.n[pv[0]], root.n_all, ' '.join([state.action2str(a) for a in pv])))\r\n",
    "\r\n",
    "        #  Return probability distribution weighted by the number of simulations\r\n",
    "        n = root = self.nodes[state.record_string()].n + 1\r\n",
    "        n = (n / np.max(n)) ** (1 / (temperature + 1e-8))\r\n",
    "        return n / n.sum()\r\n",
    "\r\n",
    "    def pv(self, state):\r\n",
    "        # Return principal variation (action sequence which is considered as the best)\r\n",
    "        s, pv_seq = copy.deepcopy(state), []\r\n",
    "        while True:\r\n",
    "            key = s.record_string()\r\n",
    "            if key not in self.nodes or self.nodes[key].n.sum() == 0:\r\n",
    "                break\r\n",
    "            best_action = sorted([(a, self.nodes[key].n[a]) for a in s.legal_actions()], key=lambda x: -x[1])[0][0]\r\n",
    "            pv_seq.append(best_action)\r\n",
    "            s.play(best_action)\r\n",
    "        return pv_seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   1 2 3\n",
      "A _ _ _\n",
      "B _ _ _\n",
      "C _ _ _\n",
      "record = \n",
      "1.01 sec. best B3. q = 0.4840. n = 31 / 91. pv = B3\n",
      "   1 2 3\n",
      "A O O _\n",
      "B _ _ _\n",
      "C X X _\n",
      "record = A1 C1 A2 C2\n",
      "1.03 sec. best C3. q = 0.2201. n = 13 / 54. pv = C3\n",
      "2.04 sec. best A3. q = 0.2105. n = 23 / 80. pv = A3\n",
      "3.04 sec. best A3. q = 0.1947. n = 30 / 99. pv = A3\n",
      "4.03 sec. best A3. q = 0.1955. n = 35 / 118. pv = A3\n",
      "5.06 sec. best A3. q = 0.1754. n = 39 / 132. pv = A3\n",
      "6.07 sec. best A3. q = 0.1391. n = 42 / 144. pv = A3\n",
      "7.07 sec. best A3. q = 0.1591. n = 43 / 156. pv = A3\n",
      "8.04 sec. best A3. q = 0.1270. n = 46 / 168. pv = A3\n",
      "9.04 sec. best A3. q = 0.1168. n = 50 / 178. pv = A3\n",
      "10.09 sec. best A3. q = 0.1291. n = 53 / 187. pv = A3\n",
      "11.06 sec. best A3. q = 0.1082. n = 54 / 198. pv = A3\n",
      "   1 2 3\n",
      "A _ X O\n",
      "B _ O O\n",
      "C X _ _\n",
      "record = B2 A2 A3 C1 B3\n",
      "1.00 sec. best A1. q = 0.0526. n = 19 / 59. pv = A1\n",
      "2.01 sec. best B1. q = 0.0370. n = 27 / 97. pv = B1\n",
      "3.03 sec. best A1. q = 0.0270. n = 37 / 132. pv = A1\n",
      "4.02 sec. best B1. q = -0.0000. n = 44 / 158. pv = B1\n",
      "5.01 sec. best A1. q = -0.0000. n = 48 / 180. pv = A1\n",
      "6.01 sec. best B1. q = -0.0000. n = 54 / 200. pv = B1\n",
      "   1 2 3\n",
      "A _ X O\n",
      "B _ O _\n",
      "C X _ _\n",
      "record = B2 A2 A3 C1\n",
      "1.02 sec. best A1. q = -0.0756. n = 12 / 50. pv = A1\n",
      "2.06 sec. best A1. q = 0.0837. n = 25 / 71. pv = A1\n",
      "3.04 sec. best A1. q = 0.0364. n = 30 / 89. pv = A1\n",
      "4.06 sec. best A1. q = 0.0364. n = 30 / 106. pv = A1\n",
      "5.04 sec. best A1. q = 0.0910. n = 34 / 119. pv = A1\n",
      "6.05 sec. best A1. q = 0.0814. n = 38 / 132. pv = A1\n",
      "7.04 sec. best A1. q = 0.0273. n = 40 / 145. pv = A1\n",
      "8.02 sec. best C2. q = 0.1778. n = 43 / 156. pv = C2\n",
      "9.10 sec. best C2. q = 0.1010. n = 46 / 167. pv = C2\n",
      "10.08 sec. best C2. q = 0.1152. n = 49 / 179. pv = C2\n",
      "11.10 sec. best C2. q = 0.1499. n = 51 / 189. pv = C2\n",
      "12.13 sec. best C2. q = 0.1901. n = 56 / 197. pv = C2\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0., 0., 0., 0., 0., 0., 0., 1., 0.], dtype=float32)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tree = Tree(Net())\r\n",
    "tree.think(State(), 100, show=True)\r\n",
    "\r\n",
    "tree = Tree(Net())\r\n",
    "tree.think(State().play('A1 C1 A2 C2'), 200, show=True)\r\n",
    "\r\n",
    "tree = Tree(Net())\r\n",
    "tree.think(State().play('B2 A2 A3 C1 B3'), 200, show=True)\r\n",
    "\r\n",
    "tree = Tree(Net())\r\n",
    "tree.think(State().play('B2 A2 A3 C1'), 200, show=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "# 定义优化器和训练流程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import paddle.optimizer as optim\r\n",
    "\r\n",
    "batch_size = 32\r\n",
    "num_steps = 100\r\n",
    "\r\n",
    "def gen_target(ep, k):\r\n",
    "    '''Generate inputs and targets for training'''\r\n",
    "    # path, reward, observation, action, policy\r\n",
    "    turn_idx = np.random.randint(len(ep[0]))\r\n",
    "    ps, vs, ax = [], [], []\r\n",
    "    for t in range(turn_idx, turn_idx + k + 1):\r\n",
    "        if t < len(ep[0]):\r\n",
    "            p = ep[4][t]\r\n",
    "            a = ep[3][t]\r\n",
    "        else: # state after finishing game\r\n",
    "            # p is 0 (loss is 0)\r\n",
    "            p = np.zeros_like(ep[4][-1])\r\n",
    "            # random action selection\r\n",
    "            a = np.zeros(np.prod(ep[3][-1].shape), dtype=np.float32)\r\n",
    "            a[np.random.randint(len(a))] = 1\r\n",
    "            a = a.reshape(ep[3][-1].shape)\r\n",
    "        vs.append([ep[1] if t % 2 == 0 else -ep[1]])\r\n",
    "        ps.append(p)\r\n",
    "        ax.append(a)\r\n",
    "        \r\n",
    "    return ep[2][turn_idx], ax, ps, vs\r\n",
    "\r\n",
    "def train(episodes, net, opt):\r\n",
    "    '''Train neural net'''\r\n",
    "    p_loss_sum, v_loss_sum = 0, 0\r\n",
    "    net.train()\r\n",
    "    k = 4\r\n",
    "    for _ in range(num_steps):\r\n",
    "        x, ax, p_target, v_target = zip(*[gen_target(episodes[np.random.randint(len(episodes))], k) for j in range(batch_size)])\r\n",
    "        x = paddle.to_tensor(np.array(x))\r\n",
    "        ax = paddle.to_tensor(np.array(ax))\r\n",
    "        p_target = paddle.to_tensor(np.array(p_target))\r\n",
    "        v_target = paddle.to_tensor(np.array(v_target),dtype='float32')\r\n",
    "\r\n",
    "        # Change the order of axis as [time step, batch, ...]\r\n",
    "        ax = paddle.transpose(ax, [1, 0, 2, 3, 4])\r\n",
    "        p_target = paddle.transpose(p_target, [1, 0, 2])\r\n",
    "        v_target = paddle.transpose(v_target, [1, 0, 2])\r\n",
    "\r\n",
    "        p_loss, v_loss = 0, 0\r\n",
    "\r\n",
    "        # Compute losses for k (+ current) steps\r\n",
    "        for t in range(k + 1):\r\n",
    "            rp = net.representation(x) if t == 0 else net.dynamics(rp, ax[t - 1])\r\n",
    "            p, v = net.prediction(rp)\r\n",
    "            p_loss += paddle.sum(-p_target[t] * paddle.log(p))\r\n",
    "            v_loss += paddle.sum((v_target[t] - v) ** 2)\r\n",
    "\r\n",
    "        p_loss_sum += paddle.squeeze(p_loss,0)\r\n",
    "        v_loss_sum += paddle.squeeze(v_loss,0)\r\n",
    "\r\n",
    "        optimizer.clear_grad()\r\n",
    "        (p_loss + v_loss).backward()\r\n",
    "        optimizer.step()\r\n",
    "\r\n",
    "    num_train_datum = num_steps * batch_size\r\n",
    "    print('p_loss %f v_loss %f' % (p_loss_sum / num_train_datum, v_loss_sum / num_train_datum))\r\n",
    "    return net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#  Battle against random agents\r\n",
    "\r\n",
    "def vs_random(net, n=100):\r\n",
    "    results = {}\r\n",
    "    for i in range(n):\r\n",
    "        first_turn = i % 2 == 0\r\n",
    "        turn = first_turn\r\n",
    "        state = State()\r\n",
    "        while not state.terminal():\r\n",
    "            if turn:\r\n",
    "                p, _ = net.predict(state, [])[-1]\r\n",
    "                action = sorted([(a, p[a]) for a in state.legal_actions()], key=lambda x:-x[1])[0][0]\r\n",
    "            else:\r\n",
    "                action = np.random.choice(state.legal_actions())\r\n",
    "            state.play(action)\r\n",
    "            turn = not turn\r\n",
    "        r = state.terminal_reward() if turn else -state.terminal_reward()\r\n",
    "        results[r] = results.get(r, 0) + 1\r\n",
    "    return results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "# 开始训练我们的神经网络"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vs_random =  [(-1, 12), (0, 11), (1, 77)]\n",
      "game 0  1  2  3  4  5  6  7  8  9  10  11  12  13  14  15  16  17  18  19  generated =  [(-1, 3), (0, 5), (1, 12)]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/nn/layer/norm.py:648: UserWarning: When training, we now always track global mean and variance.\n",
      "  \"When training, we now always track global mean and variance.\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p_loss 1.984630 v_loss 0.510406\n",
      "vs_random =  [(-1, 8), (0, 8), (1, 84)] sum =  [(-1, 20), (0, 19), (1, 161)]\n",
      "game 20  21  22  23  24  25  26  27  28  29  30  31  32  33  34  35  36  37  38  39  generated =  [(-1, 5), (0, 12), (1, 23)]\n",
      "p_loss 1.793176 v_loss 0.392811\n",
      "vs_random =  [(-1, 9), (0, 12), (1, 79)] sum =  [(-1, 29), (0, 31), (1, 240)]\n",
      "game 40  41  42  43  44  45  46  47  48  49  50  51  52  53  54  55  56  57  58  59  generated =  [(-1, 6), (0, 18), (1, 36)]\n",
      "p_loss 1.710342 v_loss 0.353093\n",
      "vs_random =  [(-1, 7), (0, 5), (1, 88)] sum =  [(-1, 36), (0, 36), (1, 328)]\n",
      "game 60  61  62  63  64  65  66  67  68  69  70  71  72  73  74  75  76  77  78  79  generated =  [(-1, 6), (0, 25), (1, 49)]\n",
      "p_loss 1.725020 v_loss 0.382014\n",
      "vs_random =  [(-1, 3), (0, 11), (1, 86)] sum =  [(-1, 39), (0, 47), (1, 414)]\n",
      "game 80  81  82  83  84  85  86  87  88  89  90  91  92  93  94  95  96  97  98  99  generated =  [(-1, 6), (0, 32), (1, 62)]\n",
      "p_loss 1.761850 v_loss 0.313605\n",
      "vs_random =  [(-1, 8), (0, 9), (1, 83)] sum =  [(-1, 47), (0, 56), (1, 497)]\n",
      "game 100  101  102  103  104  105  106  107  108  109  110  111  112  113  114  115  116  117  118  119  generated =  [(-1, 7), (0, 38), (1, 75)]\n",
      "p_loss 1.672840 v_loss 0.296055\n",
      "vs_random =  [(-1, 10), (0, 9), (1, 81)] sum =  [(-1, 57), (0, 65), (1, 578)]\n",
      "game 120  121  122  123  124  125  126  127  128  129  130  131  132  133  134  135  136  137  138  139  generated =  [(-1, 7), (0, 46), (1, 87)]\n",
      "p_loss 1.641024 v_loss 0.288286\n",
      "vs_random =  [(-1, 4), (0, 11), (1, 85)] sum =  [(-1, 61), (0, 76), (1, 663)]\n",
      "game 140  141  142  143  144  145  146  147  148  149  150  151  152  153  154  155  156  157  158  159  generated =  [(-1, 10), (0, 54), (1, 96)]\n",
      "p_loss 1.608081 v_loss 0.293587\n",
      "vs_random =  [(-1, 8), (0, 10), (1, 82)] sum =  [(-1, 69), (0, 86), (1, 745)]\n",
      "game 160  161  162  163  164  165  166  167  168  169  170  171  172  173  174  175  176  177  178  179  generated =  [(-1, 10), (0, 61), (1, 109)]\n",
      "p_loss 1.691981 v_loss 0.278807\n",
      "vs_random =  [(-1, 8), (0, 11), (1, 81)] sum =  [(-1, 77), (0, 97), (1, 826)]\n",
      "game 180  181  182  183  184  185  186  187  188  189  190  191  192  193  194  195  196  197  198  199  generated =  [(-1, 10), (0, 70), (1, 120)]\n",
      "p_loss 1.625584 v_loss 0.264618\n",
      "vs_random =  [(-1, 7), (0, 10), (1, 83)] sum =  [(-1, 84), (0, 107), (1, 909)]\n",
      "game 200  201  202  203  204  205  206  207  208  209  210  211  212  213  214  215  216  217  218  219  generated =  [(-1, 10), (0, 75), (1, 135)]\n",
      "p_loss 1.674798 v_loss 0.241374\n",
      "vs_random =  [(-1, 3), (0, 14), (1, 83)] sum =  [(-1, 87), (0, 121), (1, 992)]\n",
      "game 220  221  222  223  224  225  226  227  228  229  230  231  232  233  234  235  236  237  238  239  generated =  [(-1, 10), (0, 80), (1, 150)]\n",
      "p_loss 1.628377 v_loss 0.231433\n",
      "vs_random =  [(-1, 4), (0, 16), (1, 80)] sum =  [(-1, 91), (0, 137), (1, 1072)]\n",
      "game 240  241  242  243  244  245  246  247  248  249  250  251  252  253  254  255  256  257  258  259  generated =  [(-1, 10), (0, 86), (1, 164)]\n",
      "p_loss 1.659701 v_loss 0.227174\n",
      "vs_random =  [(-1, 4), (0, 8), (1, 88)] sum =  [(-1, 95), (0, 145), (1, 1160)]\n",
      "game 260  261  262  263  264  265  266  267  268  269  270  271  272  273  274  275  276  277  278  279  generated =  [(-1, 10), (0, 93), (1, 177)]\n",
      "p_loss 1.610759 v_loss 0.222647\n",
      "vs_random =  [(-1, 4), (0, 7), (1, 89)] sum =  [(-1, 99), (0, 152), (1, 1249)]\n",
      "game 280  281  282  283  284  285  286  287  288  289  290  291  292  293  294  295  296  297  298  299  generated =  [(-1, 10), (0, 97), (1, 193)]\n",
      "p_loss 1.642350 v_loss 0.193985\n",
      "vs_random =  [(-1, 7), (0, 9), (1, 84)] sum =  [(-1, 106), (0, 161), (1, 1333)]\n",
      "game 300  301  302  303  304  305  306  307  308  309  310  311  312  313  314  315  316  317  318  319  generated =  [(-1, 10), (0, 103), (1, 207)]\n",
      "p_loss 1.593649 v_loss 0.241393\n",
      "vs_random =  [(-1, 6), (0, 10), (1, 84)] sum =  [(-1, 112), (0, 171), (1, 1417)]\n",
      "game 320  321  322  323  324  325  326  327  328  329  330  331  332  333  334  335  336  337  338  339  generated =  [(-1, 10), (0, 107), (1, 223)]\n",
      "p_loss 1.624067 v_loss 0.235776\n",
      "vs_random =  [(-1, 3), (0, 15), (1, 82)] sum =  [(-1, 115), (0, 186), (1, 1499)]\n",
      "game 340  341  342  343  344  345  346  347  348  349  350  351  352  353  354  355  356  357  358  359  generated =  [(-1, 10), (0, 112), (1, 238)]\n",
      "p_loss 1.606524 v_loss 0.234550\n",
      "vs_random =  [(-1, 4), (0, 5), (1, 91)] sum =  [(-1, 119), (0, 191), (1, 1590)]\n",
      "game 360  361  362  363  364  365  366  367  368  369  370  371  372  373  374  375  376  377  378  379  generated =  [(-1, 10), (0, 118), (1, 252)]\n",
      "p_loss 1.620935 v_loss 0.197866\n",
      "vs_random =  [(-1, 7), (0, 9), (1, 84)] sum =  [(-1, 126), (0, 200), (1, 1674)]\n",
      "game 380  381  382  383  384  385  386  387  388  389  390  391  392  393  394  395  396  397  398  399  generated =  [(-1, 12), (0, 123), (1, 265)]\n",
      "p_loss 1.584294 v_loss 0.201287\n",
      "vs_random =  [(-1, 5), (0, 6), (1, 89)] sum =  [(-1, 131), (0, 206), (1, 1763)]\n",
      "game 400  401  402  403  404  405  406  407  408  409  410  411  412  413  414  415  416  417  418  419  generated =  [(-1, 14), (0, 128), (1, 278)]\n",
      "p_loss 1.578481 v_loss 0.190133\n",
      "vs_random =  [(-1, 6), (0, 7), (1, 87)] sum =  [(-1, 137), (0, 213), (1, 1850)]\n",
      "game 420  421  422  423  424  425  426  427  428  429  430  431  432  433  434  435  436  437  438  439  generated =  [(-1, 14), (0, 130), (1, 296)]\n",
      "p_loss 1.609202 v_loss 0.212222\n",
      "vs_random =  [(-1, 6), (0, 8), (1, 86)] sum =  [(-1, 143), (0, 221), (1, 1936)]\n",
      "game 440  441  442  443  444  445  446  447  448  449  450  451  452  453  454  455  456  457  458  459  generated =  [(-1, 14), (0, 135), (1, 311)]\n",
      "p_loss 1.585429 v_loss 0.192035\n",
      "vs_random =  [(-1, 5), (0, 5), (1, 90)] sum =  [(-1, 148), (0, 226), (1, 2026)]\n",
      "game 460  461  462  463  464  465  466  467  468  469  470  471  472  473  474  475  476  477  478  479  generated =  [(-1, 16), (0, 138), (1, 326)]\n",
      "p_loss 1.570275 v_loss 0.197700\n",
      "vs_random =  [(-1, 7), (0, 8), (1, 85)] sum =  [(-1, 155), (0, 234), (1, 2111)]\n",
      "game 480  481  482  483  484  485  486  487  488  489  490  491  492  493  494  495  496  497  498  499  generated =  [(-1, 16), (0, 142), (1, 342)]\n",
      "p_loss 1.645397 v_loss 0.221361\n",
      "vs_random =  [(-1, 5), (0, 6), (1, 89)] sum =  [(-1, 160), (0, 240), (1, 2200)]\n",
      "game 500  501  502  503  504  505  506  507  508  509  510  511  512  513  514  515  516  517  518  519  generated =  [(-1, 17), (0, 146), (1, 357)]\n",
      "p_loss 1.623874 v_loss 0.189805\n",
      "vs_random =  [(-1, 11), (0, 4), (1, 85)] sum =  [(-1, 171), (0, 244), (1, 2285)]\n",
      "game 520  521  522  523  524  525  526  527  528  529  530  531  532  533  534  535  536  537  538  539  generated =  [(-1, 17), (0, 150), (1, 373)]\n",
      "p_loss 1.642672 v_loss 0.209628\n",
      "vs_random =  [(-1, 4), (0, 7), (1, 89)] sum =  [(-1, 175), (0, 251), (1, 2374)]\n",
      "game 540  541  542  543  544  545  546  547  548  549  550  551  552  553  554  555  556  557  558  559  generated =  [(-1, 17), (0, 153), (1, 390)]\n",
      "p_loss 1.528260 v_loss 0.187127\n",
      "vs_random =  [(-1, 4), (0, 11), (1, 85)] sum =  [(-1, 179), (0, 262), (1, 2459)]\n",
      "game 560  561  562  563  564  565  566  567  568  569  570  571  572  573  574  575  576  577  578  579  generated =  [(-1, 17), (0, 163), (1, 400)]\n",
      "p_loss 1.514863 v_loss 0.190807\n",
      "vs_random =  [(-1, 4), (0, 7), (1, 89)] sum =  [(-1, 183), (0, 269), (1, 2548)]\n",
      "game 580  581  582  583  584  585  586  587  588  589  590  591  592  593  594  595  596  597  598  599  generated =  [(-1, 17), (0, 165), (1, 418)]\n",
      "p_loss 1.607478 v_loss 0.185665\n",
      "vs_random =  [(-1, 3), (0, 9), (1, 88)] sum =  [(-1, 186), (0, 278), (1, 2636)]\n",
      "game 600  601  602  603  604  605  606  607  608  609  610  611  612  613  614  615  616  617  618  619  generated =  [(-1, 17), (0, 171), (1, 432)]\n",
      "p_loss 1.594799 v_loss 0.177431\n",
      "vs_random =  [(-1, 6), (0, 4), (1, 90)] sum =  [(-1, 192), (0, 282), (1, 2726)]\n",
      "game 620  621  622  623  624  625  626  627  628  629  630  631  632  633  634  635  636  637  638  639  generated =  [(-1, 18), (0, 176), (1, 446)]\n",
      "p_loss 1.521672 v_loss 0.173386\n",
      "vs_random =  [(-1, 4), (0, 10), (1, 86)] sum =  [(-1, 196), (0, 292), (1, 2812)]\n",
      "game 640  641  642  643  644  645  646  647  648  649  650  651  652  653  654  655  656  657  658  659  generated =  [(-1, 18), (0, 179), (1, 463)]\n",
      "p_loss 1.531252 v_loss 0.148164\n",
      "vs_random =  [(-1, 3), (0, 8), (1, 89)] sum =  [(-1, 199), (0, 300), (1, 2901)]\n",
      "game 660  661  662  663  664  665  666  667  668  669  670  671  672  673  674  675  676  677  678  679  generated =  [(-1, 22), (0, 183), (1, 475)]\n",
      "p_loss 1.514773 v_loss 0.162417\n",
      "vs_random =  [(-1, 4), (0, 7), (1, 89)] sum =  [(-1, 203), (0, 307), (1, 2990)]\n",
      "game 680  681  682  683  684  685  686  687  688  689  690  691  692  693  694  695  696  697  698  699  generated =  [(-1, 23), (0, 187), (1, 490)]\n",
      "p_loss 1.501991 v_loss 0.211090\n",
      "vs_random =  [(-1, 6), (0, 6), (1, 88)] sum =  [(-1, 209), (0, 313), (1, 3078)]\n",
      "game 700  701  702  703  704  705  706  707  708  709  710  711  712  713  714  715  716  717  718  719  generated =  [(-1, 24), (0, 193), (1, 503)]\n",
      "p_loss 1.496879 v_loss 0.168363\n",
      "vs_random =  [(-1, 7), (0, 11), (1, 82)] sum =  [(-1, 216), (0, 324), (1, 3160)]\n",
      "game 720  721  722  723  724  725  726  727  728  729  730  731  732  733  734  735  736  737  738  739  generated =  [(-1, 24), (0, 194), (1, 522)]\n",
      "p_loss 1.600798 v_loss 0.183576\n",
      "vs_random =  [(-1, 12), (0, 5), (1, 83)] sum =  [(-1, 228), (0, 329), (1, 3243)]\n",
      "game 740  741  742  743  744  745  746  747  748  749  750  751  752  753  754  755  756  757  758  759  generated =  [(-1, 24), (0, 198), (1, 538)]\n",
      "p_loss 1.515208 v_loss 0.164130\n",
      "vs_random =  [(-1, 6), (0, 1), (1, 93)] sum =  [(-1, 234), (0, 330), (1, 3336)]\n",
      "game 760  761  762  763  764  765  766  767  768  769  770  771  772  773  774  775  776  777  778  779  generated =  [(-1, 24), (0, 202), (1, 554)]\n",
      "p_loss 1.534758 v_loss 0.178581\n",
      "vs_random =  [(-1, 2), (0, 7), (1, 91)] sum =  [(-1, 236), (0, 337), (1, 3427)]\n",
      "game 780  781  782  783  784  785  786  787  788  789  790  791  792  793  794  795  796  797  798  799  generated =  [(-1, 24), (0, 205), (1, 571)]\n",
      "p_loss 1.501029 v_loss 0.182956\n",
      "vs_random =  [(-1, 8), (0, 6), (1, 86)] sum =  [(-1, 244), (0, 343), (1, 3513)]\n",
      "game 800  801  802  803  804  805  806  807  808  809  810  811  812  813  814  815  816  817  818  819  generated =  [(-1, 25), (0, 209), (1, 586)]\n",
      "p_loss 1.517597 v_loss 0.224809\n",
      "vs_random =  [(-1, 9), (0, 12), (1, 79)] sum =  [(-1, 253), (0, 355), (1, 3592)]\n",
      "game 820  821  822  823  824  825  826  827  828  829  830  831  832  833  834  835  836  837  838  839  generated =  [(-1, 26), (0, 216), (1, 598)]\n",
      "p_loss 1.531028 v_loss 0.169736\n",
      "vs_random =  [(-1, 5), (0, 8), (1, 87)] sum =  [(-1, 258), (0, 363), (1, 3679)]\n",
      "game 840  841  842  843  844  845  846  847  848  849  850  851  852  853  854  855  856  857  858  859  generated =  [(-1, 26), (0, 219), (1, 615)]\n",
      "p_loss 1.529084 v_loss 0.165891\n",
      "vs_random =  [(-1, 3), (0, 9), (1, 88)] sum =  [(-1, 261), (0, 372), (1, 3767)]\n",
      "game 860  861  862  863  864  865  866  867  868  869  870  871  872  873  874  875  876  877  878  879  generated =  [(-1, 26), (0, 224), (1, 630)]\n",
      "p_loss 1.544090 v_loss 0.168982\n",
      "vs_random =  [(-1, 8), (0, 9), (1, 83)] sum =  [(-1, 269), (0, 381), (1, 3850)]\n",
      "game 880  881  882  883  884  885  886  887  888  889  890  891  892  893  894  895  896  897  898  899  generated =  [(-1, 26), (0, 228), (1, 646)]\n",
      "p_loss 1.534301 v_loss 0.177807\n",
      "vs_random =  [(-1, 8), (0, 5), (1, 87)] sum =  [(-1, 277), (0, 386), (1, 3937)]\n",
      "game 900  901  902  903  904  905  906  907  908  909  910  911  912  913  914  915  916  917  918  919  generated =  [(-1, 26), (0, 234), (1, 660)]\n",
      "p_loss 1.527440 v_loss 0.158558\n",
      "vs_random =  [(-1, 7), (0, 6), (1, 87)] sum =  [(-1, 284), (0, 392), (1, 4024)]\n",
      "game 920  921  922  923  924  925  926  927  928  929  930  931  932  933  934  935  936  937  938  939  generated =  [(-1, 26), (0, 242), (1, 672)]\n",
      "p_loss 1.493012 v_loss 0.152967\n",
      "vs_random =  [(-1, 5), (0, 6), (1, 89)] sum =  [(-1, 289), (0, 398), (1, 4113)]\n",
      "game 940  941  942  943  944  945  946  947  948  949  950  951  952  953  954  955  956  957  958  959  generated =  [(-1, 27), (0, 246), (1, 687)]\n",
      "p_loss 1.574271 v_loss 0.172361\n",
      "vs_random =  [(-1, 7), (0, 5), (1, 88)] sum =  [(-1, 296), (0, 403), (1, 4201)]\n",
      "game 960  961  962  963  964  965  966  967  968  969  970  971  972  973  974  975  976  977  978  979  generated =  [(-1, 27), (0, 251), (1, 702)]\n",
      "p_loss 1.540451 v_loss 0.181847\n",
      "vs_random =  [(-1, 8), (0, 8), (1, 84)] sum =  [(-1, 304), (0, 411), (1, 4285)]\n",
      "game 980  981  982  983  984  985  986  987  988  989  990  991  992  993  994  995  996  997  998  999  generated =  [(-1, 28), (0, 257), (1, 715)]\n",
      "p_loss 1.525429 v_loss 0.169395\n",
      "vs_random =  [(-1, 8), (0, 9), (1, 83)] sum =  [(-1, 312), (0, 420), (1, 4368)]\n",
      "game 1000  1001  1002  1003  1004  1005  1006  1007  1008  1009  1010  1011  1012  1013  1014  1015  1016  1017  1018  1019  generated =  [(-1, 29), (0, 260), (1, 731)]\n",
      "p_loss 1.580678 v_loss 0.173513\n",
      "vs_random =  [(-1, 4), (0, 8), (1, 88)] sum =  [(-1, 316), (0, 428), (1, 4456)]\n",
      "game 1020  1021  1022  1023  1024  1025  1026  1027  1028  1029  1030  1031  1032  1033  1034  1035  1036  1037  1038  1039  generated =  [(-1, 29), (0, 263), (1, 748)]\n",
      "p_loss 1.502888 v_loss 0.166755\n",
      "vs_random =  [(-1, 5), (0, 7), (1, 88)] sum =  [(-1, 321), (0, 435), (1, 4544)]\n",
      "game 1040  1041  1042  1043  1044  1045  1046  1047  1048  1049  1050  1051  1052  1053  1054  1055  1056  1057  1058  1059  generated =  [(-1, 29), (0, 269), (1, 762)]\n",
      "p_loss 1.565662 v_loss 0.163569\n",
      "vs_random =  [(-1, 11), (0, 10), (1, 79)] sum =  [(-1, 332), (0, 445), (1, 4623)]\n",
      "game 1060  1061  1062  1063  1064  1065  1066  1067  1068  1069  1070  1071  1072  1073  1074  1075  1076  1077  1078  1079  generated =  [(-1, 29), (0, 275), (1, 776)]\n",
      "p_loss 1.492792 v_loss 0.169060\n",
      "vs_random =  [(-1, 3), (0, 7), (1, 90)] sum =  [(-1, 335), (0, 452), (1, 4713)]\n",
      "game 1080  1081  1082  1083  1084  1085  1086  1087  1088  1089  1090  1091  1092  1093  1094  1095  1096  1097  1098  1099  generated =  [(-1, 29), (0, 287), (1, 784)]\n",
      "p_loss 1.504190 v_loss 0.190512\n",
      "vs_random =  [(-1, 5), (0, 8), (1, 87)] sum =  [(-1, 340), (0, 460), (1, 4800)]\n",
      "game 1100  1101  1102  1103  1104  1105  1106  1107  1108  1109  1110  1111  1112  1113  1114  1115  1116  1117  1118  1119  generated =  [(-1, 29), (0, 292), (1, 799)]\n",
      "p_loss 1.446341 v_loss 0.146341\n",
      "vs_random =  [(-1, 6), (0, 6), (1, 88)] sum =  [(-1, 346), (0, 466), (1, 4888)]\n",
      "game 1120  1121  1122  1123  1124  1125  1126  1127  1128  1129  1130  1131  1132  1133  1134  1135  1136  1137  1138  1139  generated =  [(-1, 29), (0, 297), (1, 814)]\n",
      "p_loss 1.508812 v_loss 0.149923\n",
      "vs_random =  [(-1, 3), (0, 8), (1, 89)] sum =  [(-1, 349), (0, 474), (1, 4977)]\n",
      "game 1140  1141  1142  1143  1144  1145  1146  1147  1148  1149  1150  1151  1152  1153  1154  1155  1156  1157  1158  1159  generated =  [(-1, 29), (0, 302), (1, 829)]\n",
      "p_loss 1.524691 v_loss 0.175361\n",
      "vs_random =  [(-1, 3), (0, 7), (1, 90)] sum =  [(-1, 352), (0, 481), (1, 5067)]\n",
      "game 1160  1161  1162  1163  1164  1165  1166  1167  1168  1169  1170  1171  1172  1173  1174  1175  1176  1177  1178  1179  generated =  [(-1, 31), (0, 304), (1, 845)]\n",
      "p_loss 1.561200 v_loss 0.170017\n",
      "vs_random =  [(-1, 5), (0, 3), (1, 92)] sum =  [(-1, 357), (0, 484), (1, 5159)]\n",
      "game 1180  1181  1182  1183  1184  1185  1186  1187  1188  1189  1190  1191  1192  1193  1194  1195  1196  1197  1198  1199  generated =  [(-1, 31), (0, 312), (1, 857)]\n",
      "p_loss 1.549685 v_loss 0.177778\n",
      "vs_random =  [(-1, 6), (0, 4), (1, 90)] sum =  [(-1, 363), (0, 488), (1, 5249)]\n",
      "game 1200  1201  1202  1203  1204  1205  1206  1207  1208  1209  1210  1211  1212  1213  1214  1215  1216  1217  1218  1219  generated =  [(-1, 32), (0, 317), (1, 871)]\n",
      "p_loss 1.528916 v_loss 0.180232\n",
      "vs_random =  [(-1, 6), (0, 6), (1, 88)] sum =  [(-1, 369), (0, 494), (1, 5337)]\n",
      "game 1220  1221  1222  1227  1228  1229  1230  1231  1232  1233  1234  1235  1236  1237  1238  1239  generated =  [(-1, 33), (0, 325), (1, 882)]\n",
      "p_loss 1.570236 v_loss 0.179289\n",
      "vs_random =  [(-1, 5), (0, 10), (1, 85)] sum =  [(-1, 374), (0, 504), (1, 5422)]\n",
      "game 1240  1241  1242  1243  1244  1245  1246  1247  1248  1249  1250  1251  1252  1253  1254  1255  1256  1257  1258  1259  generated =  [(-1, 33), (0, 329), (1, 898)]\n",
      "p_loss 1.490388 v_loss 0.200446\n",
      "vs_random =  [(-1, 3), (0, 10), (1, 87)] sum =  [(-1, 377), (0, 514), (1, 5509)]\n",
      "game 1260  1261  1262  1263  1264  1265  1266  1267  1268  1269  1270  1271  1272  1273  1274  1275  1276  1277  1278  1279  generated =  [(-1, 34), (0, 335), (1, 911)]\n",
      "p_loss 1.596161 v_loss 0.213007\n",
      "vs_random =  [(-1, 6), (0, 8), (1, 86)] sum =  [(-1, 383), (0, 522), (1, 5595)]\n",
      "game 1280  1281  1282  1283  1284  1285  1286  1287  1288  1289  1290  1291  1292  1293  1294  1295  1296  1297  1298  1299  generated =  [(-1, 34), (0, 340), (1, 926)]\n",
      "p_loss 1.573869 v_loss 0.202410\n",
      "vs_random =  [(-1, 4), (0, 4), (1, 92)] sum =  [(-1, 387), (0, 526), (1, 5687)]\n",
      "game 1300  1301  1302  1303  1304  1305  1306  1307  1308  1309  1310  1311  1312  1313  1314  1315  1316  1317  1318  1319  generated =  [(-1, 35), (0, 346), (1, 939)]\n",
      "p_loss 1.527464 v_loss 0.172332\n",
      "vs_random =  [(-1, 8), (0, 4), (1, 88)] sum =  [(-1, 395), (0, 530), (1, 5775)]\n",
      "game 1320  1321  1322  1323  1324  1325  1326  1327  1328  1329  1330  1331  1332  1333  1334  1335  1336  1337  1338  1339  generated =  [(-1, 35), (0, 353), (1, 952)]\n",
      "p_loss 1.528523 v_loss 0.212990\n",
      "vs_random =  [(-1, 4), (0, 9), (1, 87)] sum =  [(-1, 399), (0, 539), (1, 5862)]\n",
      "game 1340  1341  1342  1343  1344  1345  1346  1347  1348  1349  1350  1351  1352  1353  1354  1355  1356  1357  1358  1359  generated =  [(-1, 36), (0, 360), (1, 964)]\n",
      "p_loss 1.473486 v_loss 0.227584\n",
      "vs_random =  [(-1, 3), (0, 8), (1, 89)] sum =  [(-1, 402), (0, 547), (1, 5951)]\n",
      "game 1360  1361  1362  1363  1364  1365  1366  1367  1368  1369  1370  1371  1372  1373  1374  1375  1376  1377  1378  1379  generated =  [(-1, 36), (0, 367), (1, 977)]\n",
      "p_loss 1.518003 v_loss 0.227333\n",
      "vs_random =  [(-1, 6), (0, 8), (1, 86)] sum =  [(-1, 408), (0, 555), (1, 6037)]\n",
      "game 1380  1381  1382  1383  1384  1385  1386  1387  1388  1389  1390  1391  1392  1393  1394  1395  1396  1397  1398  1399  generated =  [(-1, 36), (0, 372), (1, 992)]\n",
      "p_loss 1.515274 v_loss 0.236559\n",
      "vs_random =  [(-1, 9), (0, 6), (1, 85)] sum =  [(-1, 417), (0, 561), (1, 6122)]\n",
      "game 1400  1401  1402  1403  1404  1405  1406  1407  1408  1409  1410  1411  1412  1413  1414  1415  1416  1417  1418  1419  generated =  [(-1, 37), (0, 375), (1, 1008)]\n",
      "p_loss 1.529484 v_loss 0.189253\n",
      "vs_random =  [(-1, 6), (0, 4), (1, 90)] sum =  [(-1, 423), (0, 565), (1, 6212)]\n",
      "game 1420  1421  1422  1423  1424  1425  1426  1427  1428  1429  1430  1431  1432  1433  1434  1435  1436  1437  1438  1439  generated =  [(-1, 38), (0, 380), (1, 1022)]\n",
      "p_loss 1.564604 v_loss 0.196520\n",
      "vs_random =  [(-1, 8), (0, 10), (1, 82)] sum =  [(-1, 431), (0, 575), (1, 6294)]\n",
      "game 1440  1441  1442  1443  1444  1445  1446  1447  1448  1449  1450  1451  1452  1453  1454  1455  1456  1457  1458  1459  generated =  [(-1, 39), (0, 386), (1, 1035)]\n",
      "p_loss 1.515981 v_loss 0.205940\n",
      "vs_random =  [(-1, 5), (0, 12), (1, 83)] sum =  [(-1, 436), (0, 587), (1, 6377)]\n",
      "game 1460  1461  1462  1463  1464  1465  1466  1467  1468  1469  1470  1471  1472  1473  1474  1475  1476  1477  1478  1479  generated =  [(-1, 39), (0, 392), (1, 1049)]\n",
      "p_loss 1.553097 v_loss 0.241575\n",
      "vs_random =  [(-1, 7), (0, 11), (1, 82)] sum =  [(-1, 443), (0, 598), (1, 6459)]\n",
      "game 1480  1481  1482  1483  1484  1485  1486  1487  1488  1489  1490  1491  1492  1493  1494  1495  1496  1497  1498  1499  generated =  [(-1, 39), (0, 397), (1, 1064)]\n",
      "p_loss 1.569917 v_loss 0.223773\n",
      "vs_random =  [(-1, 12), (0, 2), (1, 86)] sum =  [(-1, 455), (0, 600), (1, 6545)]\n",
      "game 1500  1501  1502  1503  1504  1505  1506  1507  1508  1509  1510  1511  1512  1513  1514  1515  1516  1517  1518  1519  generated =  [(-1, 39), (0, 405), (1, 1076)]\n",
      "p_loss 1.581677 v_loss 0.231511\n",
      "vs_random =  [(-1, 6), (0, 5), (1, 89)] sum =  [(-1, 461), (0, 605), (1, 6634)]\n",
      "game 1520  1521  1522  1523  1524  1525  1526  1527  1528  1529  1530  1531  1532  1533  1534  1535  1536  1537  1538  1539  generated =  [(-1, 39), (0, 410), (1, 1091)]\n",
      "p_loss 1.496362 v_loss 0.218404\n",
      "vs_random =  [(-1, 8), (0, 6), (1, 86)] sum =  [(-1, 469), (0, 611), (1, 6720)]\n",
      "game 1540  1541  1542  1543  1544  1545  1546  1547  1548  1549  1550  1551  1552  1553  1554  1555  1556  1557  1558  1559  generated =  [(-1, 39), (0, 417), (1, 1104)]\n",
      "p_loss 1.454122 v_loss 0.223727\n",
      "vs_random =  [(-1, 3), (0, 6), (1, 91)] sum =  [(-1, 472), (0, 617), (1, 6811)]\n",
      "game 1560  1561  1562  1563  1564  1565  1566  1567  1568  1569  1570  1571  1572  1573  1574  1575  1576  1577  1578  1579  generated =  [(-1, 40), (0, 421), (1, 1119)]\n",
      "p_loss 1.554677 v_loss 0.199829\n",
      "vs_random =  [(-1, 6), (0, 11), (1, 83)] sum =  [(-1, 478), (0, 628), (1, 6894)]\n",
      "game 1580  1581  1582  1583  1584  1585  1586  1587  1588  1589  1590  1591  1592  1593  1594  1595  1596  1597  1598  1599  generated =  [(-1, 40), (0, 424), (1, 1136)]\n",
      "p_loss 1.597496 v_loss 0.215900\n",
      "vs_random =  [(-1, 6), (0, 2), (1, 92)] sum =  [(-1, 484), (0, 630), (1, 6986)]\n",
      "game 1600  1601  1602  1603  1604  1605  1606  1607  1608  1609  1610  1611  1612  1613  1614  1615  1616  1617  1618  1619  generated =  [(-1, 40), (0, 431), (1, 1149)]\n",
      "p_loss 1.500511 v_loss 0.236859\n",
      "vs_random =  [(-1, 6), (0, 8), (1, 86)] sum =  [(-1, 490), (0, 638), (1, 7072)]\n",
      "game 1620  1621  1622  1623  1624  1625  1626  1627  1628  1629  1630  1631  1632  1633  1634  1635  1636  1637  1638  1639  generated =  [(-1, 41), (0, 435), (1, 1164)]\n",
      "p_loss 1.509314 v_loss 0.227880\n",
      "vs_random =  [(-1, 6), (0, 7), (1, 87)] sum =  [(-1, 496), (0, 645), (1, 7159)]\n",
      "game 1640  1641  1642  1643  1644  1645  1646  1647  1648  1649  1650  1651  1652  1653  1654  1655  1656  1657  1658  1659  generated =  [(-1, 41), (0, 442), (1, 1177)]\n",
      "p_loss 1.590340 v_loss 0.270970\n",
      "vs_random =  [(-1, 6), (0, 8), (1, 86)] sum =  [(-1, 502), (0, 653), (1, 7245)]\n",
      "game 1660  1661  1662  1663  1664  1665  1666  1667  1668  1669  1670  1671  1672  1673  1674  1675  1676  1677  1678  1679  generated =  [(-1, 41), (0, 451), (1, 1188)]\n",
      "p_loss 1.547422 v_loss 0.253696\n",
      "vs_random =  [(-1, 8), (0, 9), (1, 83)] sum =  [(-1, 510), (0, 662), (1, 7328)]\n",
      "game 1680  1681  1682  1683  1684  1685  1686  1687  1688  1689  1690  1691  1692  1693  1694  1695  1696  1697  1698  1699  generated =  [(-1, 41), (0, 457), (1, 1202)]\n",
      "p_loss 1.565215 v_loss 0.219067\n",
      "vs_random =  [(-1, 5), (0, 12), (1, 83)] sum =  [(-1, 515), (0, 674), (1, 7411)]\n",
      "game 1700  1701  1702  1703  1704  1705  1706  1707  1708  1709  1710  1711  1712  1713  1714  1715  1716  1717  1718  1719  generated =  [(-1, 41), (0, 465), (1, 1214)]\n",
      "p_loss 1.550860 v_loss 0.240888\n",
      "vs_random =  [(-1, 6), (0, 9), (1, 85)] sum =  [(-1, 521), (0, 683), (1, 7496)]\n",
      "game 1720  1721  1722  1723  1724  1725  1726  1727  1728  1729  1730  1731  1732  1733  1734  1735  1736  1737  1738  1739  generated =  [(-1, 41), (0, 476), (1, 1223)]\n",
      "p_loss 1.544508 v_loss 0.233226\n",
      "vs_random =  [(-1, 5), (0, 3), (1, 92)] sum =  [(-1, 526), (0, 686), (1, 7588)]\n",
      "game 1740  1741  1742  1743  1744  1745  1746  1747  1748  1749  1750  1751  1752  1753  1754  1755  1756  1757  1758  1759  generated =  [(-1, 42), (0, 482), (1, 1236)]\n",
      "p_loss 1.580167 v_loss 0.236884\n",
      "vs_random =  [(-1, 3), (0, 5), (1, 92)] sum =  [(-1, 529), (0, 691), (1, 7680)]\n",
      "game 1760  1761  1762  1763  1764  1765  1766  1767  1768  1769  1770  1771  1772  1773  1774  1775  1776  1777  1778  1779  generated =  [(-1, 45), (0, 486), (1, 1249)]\n",
      "p_loss 1.539281 v_loss 0.245866\n",
      "vs_random =  [(-1, 8), (0, 6), (1, 86)] sum =  [(-1, 537), (0, 697), (1, 7766)]\n",
      "game 1780  1781  1782  1783  1784  1785  1786  1787  1788  1789  1790  1791  1792  1793  1794  1795  1796  1797  1798  1799  generated =  [(-1, 45), (0, 494), (1, 1261)]\n",
      "p_loss 1.524062 v_loss 0.275317\n",
      "vs_random =  [(-1, 10), (0, 9), (1, 81)] sum =  [(-1, 547), (0, 706), (1, 7847)]\n",
      "game 1800  1801  1802  1803  1804  1805  1806  1807  1808  1809  1810  1811  1812  1813  1814  1815  1816  1817  1818  1819  generated =  [(-1, 48), (0, 501), (1, 1271)]\n",
      "p_loss 1.544814 v_loss 0.269285\n",
      "vs_random =  [(-1, 8), (0, 5), (1, 87)] sum =  [(-1, 555), (0, 711), (1, 7934)]\n",
      "game 1820  1821  1822  1823  1824  1825  1826  1827  1828  1829  1830  1831  1832  1833  1834  1835  1836  1837  1838  1839  generated =  [(-1, 48), (0, 506), (1, 1286)]\n",
      "p_loss 1.551164 v_loss 0.294944\n",
      "vs_random =  [(0, 10), (1, 90)] sum =  [(-1, 555), (0, 721), (1, 8024)]\n",
      "game 1840  1841  1842  1843  1844  1845  1846  1847  1848  1849  1850  1851  1852  1853  1854  1855  1856  1857  1858  1859  generated =  [(-1, 48), (0, 511), (1, 1301)]\n",
      "p_loss 1.609642 v_loss 0.304316\n",
      "vs_random =  [(-1, 4), (0, 6), (1, 90)] sum =  [(-1, 559), (0, 727), (1, 8114)]\n",
      "game 1860  1861  1862  1863  1864  1865  1866  1867  1868  1869  1870  1871  1872  1873  1874  1875  1876  1877  1878  1879  generated =  [(-1, 49), (0, 513), (1, 1318)]\n",
      "p_loss 1.542121 v_loss 0.265876\n",
      "vs_random =  [(-1, 6), (0, 9), (1, 85)] sum =  [(-1, 565), (0, 736), (1, 8199)]\n",
      "game 1880  1881  1882  1883  1884  1885  1886  1887  1888  1889  1890  1891  1892  1893  1894  1895  1896  1897  1898  1899  generated =  [(-1, 49), (0, 519), (1, 1332)]\n",
      "p_loss 1.560586 v_loss 0.263164\n",
      "vs_random =  [(-1, 3), (0, 11), (1, 86)] sum =  [(-1, 568), (0, 747), (1, 8285)]\n",
      "game 1900  1901  1902  1903  1904  1905  1906  1907  1908  1909  1910  1911  1912  1913  1914  1915  1916  1917  1918  1919  generated =  [(-1, 50), (0, 523), (1, 1347)]\n",
      "p_loss 1.568218 v_loss 0.261032\n",
      "vs_random =  [(-1, 3), (0, 5), (1, 92)] sum =  [(-1, 571), (0, 752), (1, 8377)]\n",
      "game 1920  1921  1922  1923  1924  1925  1926  1927  1928  1929  1930  1931  1932  1933  1934  1935  1936  1937  1938  1939  generated =  [(-1, 50), (0, 530), (1, 1360)]\n",
      "p_loss 1.563040 v_loss 0.261761\n",
      "vs_random =  [(-1, 5), (0, 4), (1, 91)] sum =  [(-1, 576), (0, 756), (1, 8468)]\n",
      "game 1940  1941  1942  1943  1944  1945  1946  1947  1948  1949  1950  1951  1952  1953  1954  1955  1956  1957  1958  1959  generated =  [(-1, 52), (0, 536), (1, 1372)]\n",
      "p_loss 1.558284 v_loss 0.250100\n",
      "vs_random =  [(-1, 1), (0, 4), (1, 95)] sum =  [(-1, 577), (0, 760), (1, 8563)]\n",
      "game 1960  1961  1962  1963  1964  1965  1966  1967  1968  1969  1970  1971  1972  1973  1974  1975  1976  1977  1978  1979  generated =  [(-1, 53), (0, 544), (1, 1383)]\n",
      "p_loss 1.524908 v_loss 0.289615\n",
      "vs_random =  [(-1, 2), (0, 7), (1, 91)] sum =  [(-1, 579), (0, 767), (1, 8654)]\n",
      "game 1980  1981  1982  1983  1984  1985  1986  1987  1988  1989  1990  1991  1992  1993  1994  1995  1996  1997  1998  1999  generated =  [(-1, 53), (0, 552), (1, 1395)]\n",
      "p_loss 1.570720 v_loss 0.277302\n",
      "vs_random =  [(-1, 6), (0, 5), (1, 89)] sum =  [(-1, 585), (0, 772), (1, 8743)]\n",
      "game 2000  2001  2002  2003  2004  2005  2006  2007  2008  2009  2010  2011  2012  2013  2014  2015  2016  2017  2018  2019  generated =  [(-1, 55), (0, 561), (1, 1404)]\n",
      "p_loss 1.533652 v_loss 0.300137\n",
      "vs_random =  [(-1, 7), (0, 6), (1, 87)] sum =  [(-1, 592), (0, 778), (1, 8830)]\n",
      "game 2020  2021  2022  2023  2024  2025  2026  2027  2028  2029  2030  2031  2032  2033  2034  2035  2036  2037  2038  2039  generated =  [(-1, 56), (0, 569), (1, 1415)]\n",
      "p_loss 1.563541 v_loss 0.274098\n",
      "vs_random =  [(-1, 8), (0, 9), (1, 83)] sum =  [(-1, 600), (0, 787), (1, 8913)]\n",
      "game 2040  2041  2042  2043  2044  2045  2046  2047  2048  2049  2050  2051  2052  2053  2054  2055  2056  2057  2058  2059  generated =  [(-1, 56), (0, 576), (1, 1428)]\n",
      "p_loss 1.618187 v_loss 0.306741\n",
      "vs_random =  [(-1, 2), (0, 3), (1, 95)] sum =  [(-1, 602), (0, 790), (1, 9008)]\n",
      "game 2060  2061  2062  2063  2064  2065  2066  2067  2068  2069  2070  2071  2072  2073  2074  2075  2076  2077  2078  2079  generated =  [(-1, 57), (0, 583), (1, 1440)]\n",
      "p_loss 1.554739 v_loss 0.264270\n",
      "vs_random =  [(-1, 7), (0, 7), (1, 86)] sum =  [(-1, 609), (0, 797), (1, 9094)]\n",
      "game 2080  2081  2082  2083  2084  2085  2086  2087  2088  2089  2090  2091  2092  2093  2094  2095  2096  2097  2098  2099  generated =  [(-1, 57), (0, 590), (1, 1453)]\n",
      "p_loss 1.521998 v_loss 0.270507\n",
      "vs_random =  [(-1, 6), (0, 9), (1, 85)] sum =  [(-1, 615), (0, 806), (1, 9179)]\n",
      "game 2100  2101  2102  2103  2104  2105  2106  2107  2108  2109  2110  2111  2112  2113  2114  2115  2116  2117  2118  2119  generated =  [(-1, 57), (0, 596), (1, 1467)]\n",
      "p_loss 1.604588 v_loss 0.295330\n",
      "vs_random =  [(-1, 2), (0, 8), (1, 90)] sum =  [(-1, 617), (0, 814), (1, 9269)]\n",
      "game 2120  2121  2122  2123  2124  2125  2126  2127  2128  2129  2130  2131  2132  2133  2134  2135  2136  2137  2138  2139  generated =  [(-1, 57), (0, 603), (1, 1480)]\n",
      "p_loss 1.564728 v_loss 0.282295\n",
      "vs_random =  [(-1, 5), (0, 11), (1, 84)] sum =  [(-1, 622), (0, 825), (1, 9353)]\n",
      "game 2140  2141  2142  2143  2144  2145  2146  2147  2148  2149  2150  2151  2152  2153  2154  2155  2156  2157  2158  2159  generated =  [(-1, 58), (0, 613), (1, 1489)]\n",
      "p_loss 1.544180 v_loss 0.270897\n",
      "vs_random =  [(-1, 5), (0, 5), (1, 90)] sum =  [(-1, 627), (0, 830), (1, 9443)]\n",
      "game 2160  2161  2162  2163  2164  2165  2166  2167  2168  2169  2170  2171  2172  2173  2174  2175  2176  2177  2178  2179  generated =  [(-1, 59), (0, 621), (1, 1500)]\n",
      "p_loss 1.580577 v_loss 0.284765\n",
      "vs_random =  [(-1, 3), (0, 7), (1, 90)] sum =  [(-1, 630), (0, 837), (1, 9533)]\n",
      "game 2180  2181  2182  2183  2184  2185  2186  2187  2188  2189  2190  2191  2192  2193  2194  2195  2196  2197  2198  2199  generated =  [(-1, 59), (0, 629), (1, 1512)]\n",
      "p_loss 1.563150 v_loss 0.293585\n",
      "vs_random =  [(-1, 5), (0, 8), (1, 87)] sum =  [(-1, 635), (0, 845), (1, 9620)]\n",
      "game 2200  2201  2202  2203  2204  2205  2206  2207  2208  2209  2210  2211  2212  2213  2214  2215  2216  2217  2218  2219  generated =  [(-1, 59), (0, 636), (1, 1525)]\n",
      "p_loss 1.573802 v_loss 0.261691\n",
      "vs_random =  [(-1, 4), (0, 4), (1, 92)] sum =  [(-1, 639), (0, 849), (1, 9712)]\n",
      "game 2220  2221  2222  2223  2224  2225  2226  2227  2228  2229  2230  2231  2232  2233  2234  2235  2236  2237  2238  2239  generated =  [(-1, 59), (0, 646), (1, 1535)]\n",
      "p_loss 1.534722 v_loss 0.285565\n",
      "vs_random =  [(-1, 3), (0, 4), (1, 93)] sum =  [(-1, 642), (0, 853), (1, 9805)]\n",
      "game 2240  2241  2242  2243  2244  2245  2246  2247  2248  2249  2250  2251  2252  2253  2254  2255  2256  2257  2258  2259  generated =  [(-1, 60), (0, 652), (1, 1548)]\n",
      "p_loss 1.556539 v_loss 0.275884\n",
      "vs_random =  [(-1, 4), (0, 9), (1, 87)] sum =  [(-1, 646), (0, 862), (1, 9892)]\n",
      "game 2260  2261  2262  2263  2264  2265  2266  2267  2268  2269  2270  2271  2272  2273  2274  2275  2276  2277  2278  2279  generated =  [(-1, 60), (0, 667), (1, 1553)]\n",
      "p_loss 1.515717 v_loss 0.254869\n",
      "vs_random =  [(-1, 4), (0, 7), (1, 89)] sum =  [(-1, 650), (0, 869), (1, 9981)]\n",
      "game 2280  2281  2282  2283  2284  2285  2286  2287  2288  2289  2290  2291  2292  2293  2294  2295  2296  2297  2298  2299  generated =  [(-1, 61), (0, 672), (1, 1567)]\n",
      "p_loss 1.552961 v_loss 0.271306\n",
      "vs_random =  [(-1, 2), (0, 5), (1, 93)] sum =  [(-1, 652), (0, 874), (1, 10074)]\n",
      "game 2300  2301  2302  2303  2304  2305  2306  2307  2308  2309  2310  2311  2312  2313  2314  2315  2316  2317  2318  2319  generated =  [(-1, 62), (0, 678), (1, 1580)]\n",
      "p_loss 1.593474 v_loss 0.315569\n",
      "vs_random =  [(-1, 6), (0, 2), (1, 92)] sum =  [(-1, 658), (0, 876), (1, 10166)]\n",
      "game 2320  2321  2322  2323  2324  2325  2326  2327  2328  2329  2330  2331  2332  2333  2334  2335  2336  2337  2338  2339  generated =  [(-1, 62), (0, 688), (1, 1590)]\n",
      "p_loss 1.600889 v_loss 0.309890\n",
      "vs_random =  [(-1, 5), (0, 11), (1, 84)] sum =  [(-1, 663), (0, 887), (1, 10250)]\n",
      "game 2340  2341  2342  2343  2344  2345  2346  2347  2348  2349  2350  2351  2352  2353  2354  2355  2356  2357  2358  2359  generated =  [(-1, 62), (0, 697), (1, 1601)]\n",
      "p_loss 1.558254 v_loss 0.261698\n",
      "vs_random =  [(-1, 9), (0, 1), (1, 90)] sum =  [(-1, 672), (0, 888), (1, 10340)]\n",
      "game 2360  2361  2362  2363  2364  2365  2366  2367  2368  2369  2370  2371  2372  2373  2374  2375  2376  2377  2378  2379  generated =  [(-1, 62), (0, 708), (1, 1610)]\n",
      "p_loss 1.522512 v_loss 0.281847\n",
      "vs_random =  [(-1, 4), (0, 2), (1, 94)] sum =  [(-1, 676), (0, 890), (1, 10434)]\n",
      "game 2380  2381  2382  2383  2384  2385  2386  2387  2388  2389  2390  2391  2392  2393  2394  2395  2396  2397  2398  2399  generated =  [(-1, 62), (0, 717), (1, 1621)]\n",
      "p_loss 1.601561 v_loss 0.267094\n",
      "vs_random =  [(-1, 1), (0, 2), (1, 97)] sum =  [(-1, 677), (0, 892), (1, 10531)]\n",
      "game 2400  2401  2402  2403  2404  2405  2406  2407  2408  2409  2410  2411  2412  2413  2414  2415  2416  2417  2418  2419  generated =  [(-1, 62), (0, 729), (1, 1629)]\n",
      "p_loss 1.550334 v_loss 0.257820\n",
      "vs_random =  [(-1, 4), (0, 3), (1, 93)] sum =  [(-1, 681), (0, 895), (1, 10624)]\n",
      "game 2420  2421  2422  2423  2424  2425  2426  2427  2428  2429  2430  2431  2432  2433  2434  2435  2436  2437  2438  2439  generated =  [(-1, 63), (0, 735), (1, 1642)]\n",
      "p_loss 1.534674 v_loss 0.310985\n",
      "vs_random =  [(-1, 5), (0, 8), (1, 87)] sum =  [(-1, 686), (0, 903), (1, 10711)]\n",
      "game 2440  2441  2442  2443  2444  2445  2446  2447  2448  2449  2450  2451  2452  2453  2454  2455  2456  2457  2458  2459  generated =  [(-1, 63), (0, 739), (1, 1658)]\n",
      "p_loss 1.567424 v_loss 0.284222\n",
      "vs_random =  [(-1, 2), (0, 9), (1, 89)] sum =  [(-1, 688), (0, 912), (1, 10800)]\n",
      "game 2460  2461  2462  2463  2464  2465  2466  2467  2468  2469  2470  2471  2472  2473  2474  2475  2476  2477  2478  2479  generated =  [(-1, 64), (0, 746), (1, 1670)]\n",
      "p_loss 1.632961 v_loss 0.264902\n",
      "vs_random =  [(-1, 2), (0, 9), (1, 89)] sum =  [(-1, 690), (0, 921), (1, 10889)]\n",
      "game 2480  2481  2482  2483  2484  2485  2486  2487  2488  2489  2490  2491  2492  2493  2494  2495  2496  2497  2498  2499  generated =  [(-1, 64), (0, 752), (1, 1684)]\n",
      "p_loss 1.530500 v_loss 0.260194\n",
      "vs_random =  [(-1, 3), (0, 5), (1, 92)] sum =  [(-1, 693), (0, 926), (1, 10981)]\n",
      "game 2500  2501  2502  2503  2504  2505  2506  2507  2508  2509  2510  2511  2512  2513  2514  2515  2516  2517  2518  2519  generated =  [(-1, 64), (0, 758), (1, 1698)]\n",
      "p_loss 1.515335 v_loss 0.275010\n",
      "vs_random =  [(-1, 5), (0, 5), (1, 90)] sum =  [(-1, 698), (0, 931), (1, 11071)]\n",
      "game 2520  2521  2522  2523  2524  2525  2526  2527  2528  2529  2530  2531  2532  2533  2534  2535  2536  2537  2538  2539  generated =  [(-1, 64), (0, 763), (1, 1713)]\n",
      "p_loss 1.586563 v_loss 0.309264\n",
      "vs_random =  [(-1, 6), (0, 7), (1, 87)] sum =  [(-1, 704), (0, 938), (1, 11158)]\n",
      "game 2540  2541  2542  2543  2544  2545  2546  2547  2548  2549  2550  2551  2552  2553  2554  2555  2556  2557  2558  2559  generated =  [(-1, 65), (0, 770), (1, 1725)]\n",
      "p_loss 1.567713 v_loss 0.242718\n",
      "vs_random =  [(-1, 6), (0, 9), (1, 85)] sum =  [(-1, 710), (0, 947), (1, 11243)]\n",
      "game 2560  2561  2562  2563  2564  2565  2566  2567  2568  2569  2570  2571  2572  2573  2574  2575  2576  2577  2578  2579  generated =  [(-1, 65), (0, 775), (1, 1740)]\n",
      "p_loss 1.527696 v_loss 0.276144\n",
      "vs_random =  [(-1, 6), (0, 5), (1, 89)] sum =  [(-1, 716), (0, 952), (1, 11332)]\n",
      "game 2580  2581  2582  2583  2584  2585  2586  2587  2588  2589  2590  2591  2592  2593  2594  2595  2596  2597  2598  2599  generated =  [(-1, 65), (0, 784), (1, 1751)]\n",
      "p_loss 1.569581 v_loss 0.283896\n",
      "vs_random =  [(-1, 7), (0, 6), (1, 87)] sum =  [(-1, 723), (0, 958), (1, 11419)]\n",
      "game 2600  2601  2602  2603  2604  2605  2606  2607  2608  2609  2610  2611  2612  2613  2614  2615  2616  2617  2618  2619  generated =  [(-1, 65), (0, 789), (1, 1766)]\n",
      "p_loss 1.584276 v_loss 0.286862\n",
      "vs_random =  [(-1, 6), (0, 10), (1, 84)] sum =  [(-1, 729), (0, 968), (1, 11503)]\n",
      "game 2620  2621  2622  2623  2624  2625  2626  2627  2628  2629  2630  2631  2632  2633  2634  2635  2636  2637  2638  2639  generated =  [(-1, 65), (0, 796), (1, 1779)]\n",
      "p_loss 1.579806 v_loss 0.268765\n",
      "vs_random =  [(-1, 5), (0, 9), (1, 86)] sum =  [(-1, 734), (0, 977), (1, 11589)]\n",
      "game 2640  2641  2642  2643  2644  2645  2646  2647  2648  2649  2650  2651  2652  2653  2654  2655  2656  2657  2658  2659  generated =  [(-1, 65), (0, 801), (1, 1794)]\n",
      "p_loss 1.598632 v_loss 0.259156\n",
      "vs_random =  [(-1, 5), (0, 5), (1, 90)] sum =  [(-1, 739), (0, 982), (1, 11679)]\n",
      "game 2660  2661  2662  2663  2664  2665  2666  2667  2668  2669  2670  2671  2672  2673  2674  2675  2676  2677  2678  2679  generated =  [(-1, 65), (0, 814), (1, 1801)]\n",
      "p_loss 1.568170 v_loss 0.255019\n",
      "vs_random =  [(-1, 5), (0, 6), (1, 89)] sum =  [(-1, 744), (0, 988), (1, 11768)]\n",
      "game 2680  2681  2682  2683  2684  2685  2686  2687  2688  2689  2690  2691  2692  2693  2694  2695  2696  2697  2698  2699  generated =  [(-1, 65), (0, 817), (1, 1818)]\n",
      "p_loss 1.598950 v_loss 0.270757\n",
      "vs_random =  [(-1, 4), (0, 7), (1, 89)] sum =  [(-1, 748), (0, 995), (1, 11857)]\n",
      "game 2700  2701  2702  2703  2704  2705  2706  2707  2708  2709  2710  2711  2712  2713  2714  2715  2716  2717  2718  2719  generated =  [(-1, 65), (0, 825), (1, 1830)]\n",
      "p_loss 1.527499 v_loss 0.290078\n",
      "vs_random =  [(-1, 5), (0, 5), (1, 90)] sum =  [(-1, 753), (0, 1000), (1, 11947)]\n",
      "game 2720  2721  2722  2723  2724  2725  2726  2727  2728  2729  2730  2731  2732  2733  2734  2735  2736  2737  2738  2739  generated =  [(-1, 65), (0, 833), (1, 1842)]\n",
      "p_loss 1.554483 v_loss 0.268969\n",
      "vs_random =  [(-1, 7), (0, 3), (1, 90)] sum =  [(-1, 760), (0, 1003), (1, 12037)]\n",
      "game 2740  2741  2742  2743  2744  2745  2746  2747  2748  2749  2750  2751  2752  2753  2754  2755  2756  2757  2758  2759  generated =  [(-1, 65), (0, 843), (1, 1852)]\n",
      "p_loss 1.535121 v_loss 0.236314\n",
      "vs_random =  [(-1, 6), (0, 3), (1, 91)] sum =  [(-1, 766), (0, 1006), (1, 12128)]\n",
      "game 2760  2761  2762  2763  2764  2765  2766  2767  2768  2769  2770  2771  2772  2773  2774  2775  2776  2777  2778  2779  generated =  [(-1, 65), (0, 851), (1, 1864)]\n",
      "p_loss 1.577652 v_loss 0.256683\n",
      "vs_random =  [(-1, 6), (0, 1), (1, 93)] sum =  [(-1, 772), (0, 1007), (1, 12221)]\n",
      "game 2780  2781  2782  2783  2784  2785  2786  2787  2788  2789  2790  2791  2792  2793  2794  2795  2796  2797  2798  2799  generated =  [(-1, 65), (0, 863), (1, 1872)]\n",
      "p_loss 1.582983 v_loss 0.276364\n",
      "vs_random =  [(-1, 2), (0, 11), (1, 87)] sum =  [(-1, 774), (0, 1018), (1, 12308)]\n",
      "game 2800  2801  2802  2803  2804  2805  2806  2807  2808  2809  2810  2811  2812  2813  2814  2815  2816  2817  2818  2819  generated =  [(-1, 65), (0, 876), (1, 1879)]\n",
      "p_loss 1.531454 v_loss 0.286978\n",
      "vs_random =  [(-1, 5), (0, 4), (1, 91)] sum =  [(-1, 779), (0, 1022), (1, 12399)]\n",
      "game 2820  2821  2822  2823  2824  2825  2826  2827  2828  2829  2830  2831  2832  2833  2834  2835  2836  2837  2838  2839  generated =  [(-1, 65), (0, 887), (1, 1888)]\n",
      "p_loss 1.545079 v_loss 0.274102\n",
      "vs_random =  [(-1, 3), (0, 8), (1, 89)] sum =  [(-1, 782), (0, 1030), (1, 12488)]\n",
      "game 2840  2841  2842  2843  2844  2845  2846  2847  2848  2849  2850  2851  2852  2853  2854  2855  2856  2857  2858  2859  generated =  [(-1, 65), (0, 896), (1, 1899)]\n",
      "p_loss 1.545345 v_loss 0.220316\n",
      "vs_random =  [(-1, 8), (0, 6), (1, 86)] sum =  [(-1, 790), (0, 1036), (1, 12574)]\n",
      "game 2860  2861  2862  2863  2864  2865  2866  2867  2868  2869  2870  2871  2872  2873  2874  2875  2876  2877  2878  2879  generated =  [(-1, 65), (0, 909), (1, 1906)]\n",
      "p_loss 1.556132 v_loss 0.254154\n",
      "vs_random =  [(-1, 6), (0, 9), (1, 85)] sum =  [(-1, 796), (0, 1045), (1, 12659)]\n",
      "game 2880  2881  2882  2883  2884  2885  2886  2887  2888  2889  2890  2891  2892  2893  2894  2895  2896  2897  2898  2899  generated =  [(-1, 66), (0, 919), (1, 1915)]\n",
      "p_loss 1.605208 v_loss 0.263855\n",
      "vs_random =  [(-1, 4), (0, 2), (1, 94)] sum =  [(-1, 800), (0, 1047), (1, 12753)]\n",
      "game 2900  2901  2902  2903  2904  2905  2906  2907  2908  2909  2910  2911  2912  2913  2914  2915  2916  2917  2918  2919  generated =  [(-1, 67), (0, 925), (1, 1928)]\n",
      "p_loss 1.622789 v_loss 0.279641\n",
      "vs_random =  [(-1, 6), (0, 8), (1, 86)] sum =  [(-1, 806), (0, 1055), (1, 12839)]\n",
      "game 2920  2921  2922  2923  2924  2925  2926  2927  2928  2929  2930  2931  2932  2933  2934  2935  2936  2937  2938  2939  generated =  [(-1, 67), (0, 934), (1, 1939)]\n",
      "p_loss 1.553988 v_loss 0.266694\n",
      "vs_random =  [(-1, 3), (0, 8), (1, 89)] sum =  [(-1, 809), (0, 1063), (1, 12928)]\n",
      "game 2940  2941  2942  2943  2944  2945  2946  2947  2948  2949  2950  2951  2952  2953  2954  2955  2956  2957  2958  2959  generated =  [(-1, 67), (0, 944), (1, 1949)]\n",
      "p_loss 1.561977 v_loss 0.275962\n",
      "vs_random =  [(-1, 3), (0, 4), (1, 93)] sum =  [(-1, 812), (0, 1067), (1, 13021)]\n",
      "game 2960  2961  2962  2963  2964  2965  2966  2967  2968  2969  2970  2971  2972  2973  2974  2975  2976  2977  2978  2979  generated =  [(-1, 67), (0, 952), (1, 1961)]\n",
      "p_loss 1.490818 v_loss 0.243379\n",
      "vs_random =  [(-1, 6), (0, 13), (1, 81)] sum =  [(-1, 818), (0, 1080), (1, 13102)]\n",
      "game 2980  2981  2982  2983  2984  2985  2986  2987  2988  2989  2990  2991  2992  2993  2994  2995  2996  2997  2998  2999  generated =  [(-1, 68), (0, 960), (1, 1972)]\n",
      "p_loss 1.484984 v_loss 0.268084\n",
      "vs_random =  [(-1, 5), (0, 10), (1, 85)] sum =  [(-1, 823), (0, 1090), (1, 13187)]\n",
      "game 3000  3001  3002  3003  3004  3005  3006  3007  3008  3009  3010  3011  3012  3013  3014  3015  3016  3017  3018  3019  generated =  [(-1, 69), (0, 966), (1, 1985)]\n",
      "p_loss 1.516993 v_loss 0.278526\n",
      "vs_random =  [(-1, 4), (0, 8), (1, 88)] sum =  [(-1, 827), (0, 1098), (1, 13275)]\n",
      "game 3020  3021  3022  3023  3024  3025  3026  3027  3028  3029  3030  3031  3032  3033  3034  3035  3036  3037  3038  3039  generated =  [(-1, 69), (0, 977), (1, 1994)]\n",
      "p_loss 1.497823 v_loss 0.275506\n",
      "vs_random =  [(-1, 2), (0, 7), (1, 91)] sum =  [(-1, 829), (0, 1105), (1, 13366)]\n",
      "game 3040  3041  3042  3043  3044  3045  3046  3047  3048  3049  3050  3051  3052  3053  3054  3055  3056  3057  3058  3059  generated =  [(-1, 69), (0, 985), (1, 2006)]\n",
      "p_loss 1.585740 v_loss 0.292773\n",
      "vs_random =  [(-1, 3), (0, 6), (1, 91)] sum =  [(-1, 832), (0, 1111), (1, 13457)]\n",
      "game 3060  3061  3062  3063  3064  3065  3066  3067  3068  3069  3070  3071  3072  3073  3074  3075  3076  3077  3078  3079  generated =  [(-1, 69), (0, 989), (1, 2022)]\n",
      "p_loss 1.562392 v_loss 0.267943\n",
      "vs_random =  [(-1, 6), (0, 6), (1, 88)] sum =  [(-1, 838), (0, 1117), (1, 13545)]\n",
      "game 3080  3081  3082  3083  3084  3085  3086  3087  3088  3089  3090  3091  3092  3093  3094  3095  3096  3097  3098  3099  generated =  [(-1, 69), (0, 996), (1, 2035)]\n",
      "p_loss 1.520077 v_loss 0.258027\n",
      "vs_random =  [(-1, 6), (0, 6), (1, 88)] sum =  [(-1, 844), (0, 1123), (1, 13633)]\n",
      "game 3100  3101  3102  3103  3104  3105  3106  3107  3108  3109  3110  3111  3112  3113  3114  3115  3116  3117  3118  3119  generated =  [(-1, 70), (0, 1003), (1, 2047)]\n",
      "p_loss 1.520426 v_loss 0.282737\n",
      "vs_random =  [(-1, 1), (0, 6), (1, 93)] sum =  [(-1, 845), (0, 1129), (1, 13726)]\n",
      "game 3120  3121  3122  3123  3124  3125  3126  3127  3128  3129  3130  3131  3132  3133  3134  3135  3136  3137  3138  3139  generated =  [(-1, 72), (0, 1013), (1, 2055)]\n",
      "p_loss 1.582248 v_loss 0.268340\n",
      "vs_random =  [(-1, 4), (0, 8), (1, 88)] sum =  [(-1, 849), (0, 1137), (1, 13814)]\n",
      "game 3140  3141  3142  3143  3144  3145  3146  3147  3148  3149  3150  3151  3152  3153  3154  3155  3156  3157  3158  3159  generated =  [(-1, 73), (0, 1020), (1, 2067)]\n",
      "p_loss 1.543425 v_loss 0.247222\n",
      "vs_random =  [(-1, 8), (0, 5), (1, 87)] sum =  [(-1, 857), (0, 1142), (1, 13901)]\n",
      "game 3160  3161  3162  3163  3164  3165  3166  3167  3168  3169  3170  3171  3172  3173  3174  3175  3176  3177  3178  3179  generated =  [(-1, 73), (0, 1027), (1, 2080)]\n",
      "p_loss 1.551254 v_loss 0.244623\n",
      "vs_random =  [(-1, 6), (0, 8), (1, 86)] sum =  [(-1, 863), (0, 1150), (1, 13987)]\n",
      "game 3180  3181  3182  3183  3184  3185  3186  3187  3188  3189  3190  3191  3192  3193  3194  3195  3196  3197  3198  3199  generated =  [(-1, 74), (0, 1038), (1, 2088)]\n",
      "p_loss 1.521973 v_loss 0.264427\n",
      "vs_random =  [(-1, 5), (0, 6), (1, 89)] sum =  [(-1, 868), (0, 1156), (1, 14076)]\n",
      "game 3200  3201  3202  3203  3204  3205  3206  3207  3208  3209  3210  3211  3212  3213  3214  3215  3216  3217  3218  3219  generated =  [(-1, 74), (0, 1046), (1, 2100)]\n",
      "p_loss 1.516342 v_loss 0.262302\n",
      "vs_random =  [(-1, 3), (0, 11), (1, 86)] sum =  [(-1, 871), (0, 1167), (1, 14162)]\n",
      "game 3220  3221  3222  3223  3224  3225  3226  3227  3228  3229  3230  3231  3232  3233  3234  3235  3236  3237  3238  3239  generated =  [(-1, 74), (0, 1052), (1, 2114)]\n",
      "p_loss 1.562255 v_loss 0.256876\n",
      "vs_random =  [(-1, 4), (0, 13), (1, 83)] sum =  [(-1, 875), (0, 1180), (1, 14245)]\n",
      "game 3240  3241  3242  3243  3244  3245  3246  3247  3248  3249  3250  3251  3252  3253  3254  3255  3256  3257  3258  3259  generated =  [(-1, 74), (0, 1058), (1, 2128)]\n",
      "p_loss 1.525001 v_loss 0.241613\n",
      "vs_random =  [(-1, 5), (0, 3), (1, 92)] sum =  [(-1, 880), (0, 1183), (1, 14337)]\n",
      "game 3260  3261  3262  3263  3264  3265  3266  3267  3268  3269  3270  3271  3272  3273  3274  3275  3276  3277  3278  3279  generated =  [(-1, 74), (0, 1070), (1, 2136)]\n",
      "p_loss 1.570932 v_loss 0.274909\n",
      "vs_random =  [(-1, 4), (0, 7), (1, 89)] sum =  [(-1, 884), (0, 1190), (1, 14426)]\n",
      "game 3280  3281  3282  3283  3284  3285  3286  3287  3288  3289  3290  3291  3292  3293  3294  3295  3296  3297  3298  3299  generated =  [(-1, 74), (0, 1084), (1, 2142)]\n",
      "p_loss 1.543538 v_loss 0.254097\n",
      "vs_random =  [(-1, 2), (0, 11), (1, 87)] sum =  [(-1, 886), (0, 1201), (1, 14513)]\n",
      "game 3300  3301  3302  3303  3304  3305  3306  3307  3308  3309  3310  3311  3312  3313  3314  3315  3316  3317  3318  3319  generated =  [(-1, 74), (0, 1094), (1, 2152)]\n",
      "p_loss 1.508150 v_loss 0.264661\n",
      "vs_random =  [(-1, 7), (0, 8), (1, 85)] sum =  [(-1, 893), (0, 1209), (1, 14598)]\n",
      "game 3320  3321  3322  3323  3324  3325  3326  3327  3328  3329  3330  3331  3332  3333  3334  3335  3336  3337  3338  3339  generated =  [(-1, 74), (0, 1103), (1, 2163)]\n",
      "p_loss 1.535758 v_loss 0.240461\n",
      "vs_random =  [(-1, 3), (0, 9), (1, 88)] sum =  [(-1, 896), (0, 1218), (1, 14686)]\n",
      "game 3340  3341  3342  3343  3344  3345  3346  3347  3348  3349  3350  3351  3352  3353  3354  3355  3356  3357  3358  3359  generated =  [(-1, 74), (0, 1114), (1, 2172)]\n",
      "p_loss 1.574371 v_loss 0.264378\n",
      "vs_random =  [(0, 8), (1, 92)] sum =  [(-1, 896), (0, 1226), (1, 14778)]\n",
      "game 3360  3361  3362  3363  3364  3365  3366  3367  3368  3369  3370  3371  3372  3373  3374  3375  3376  3377  3378  3379  generated =  [(-1, 74), (0, 1124), (1, 2182)]\n",
      "p_loss 1.552863 v_loss 0.269787\n",
      "vs_random =  [(-1, 3), (0, 8), (1, 89)] sum =  [(-1, 899), (0, 1234), (1, 14867)]\n",
      "game 3380  3381  3382  3383  3384  3385  3386  3387  3388  3389  3390  3391  3392  3393  3394  3395  3396  3397  3398  3399  generated =  [(-1, 74), (0, 1133), (1, 2193)]\n",
      "p_loss 1.601708 v_loss 0.283362\n",
      "vs_random =  [(-1, 7), (0, 5), (1, 88)] sum =  [(-1, 906), (0, 1239), (1, 14955)]\n",
      "game 3400  3401  3402  3403  3404  3405  3406  3407  3408  3409  3410  3411  3412  3413  3414  3415  3416  3417  3418  3419  generated =  [(-1, 75), (0, 1139), (1, 2206)]\n",
      "p_loss 1.564255 v_loss 0.271013\n",
      "vs_random =  [(-1, 2), (0, 4), (1, 94)] sum =  [(-1, 908), (0, 1243), (1, 15049)]\n",
      "game 3420  3421  3422  3423  3424  3425  3426  3427  3428  3429  3430  3431  3432  3433  3434  3435  3436  3437  3438  3439  generated =  [(-1, 75), (0, 1145), (1, 2220)]\n",
      "p_loss 1.631953 v_loss 0.262943\n",
      "vs_random =  [(-1, 4), (0, 5), (1, 91)] sum =  [(-1, 912), (0, 1248), (1, 15140)]\n",
      "game 3440  3441  3442  3443  3444  3445  3446  3447  3448  3449  3450  3451  3452  3453  3454  3455  3456  3457  3458  3459  generated =  [(-1, 75), (0, 1153), (1, 2232)]\n",
      "p_loss 1.561633 v_loss 0.294471\n",
      "vs_random =  [(-1, 2), (0, 9), (1, 89)] sum =  [(-1, 914), (0, 1257), (1, 15229)]\n",
      "game 3460  3461  3462  3463  3464  3465  3466  3467  3468  3469  3470  3471  3472  3473  3474  3475  3476  3477  3478  3479  generated =  [(-1, 77), (0, 1161), (1, 2242)]\n",
      "p_loss 1.459340 v_loss 0.268680\n",
      "vs_random =  [(-1, 7), (0, 7), (1, 86)] sum =  [(-1, 921), (0, 1264), (1, 15315)]\n",
      "game 3480  3481  3482  3483  3484  3485  3486  3487  3488  3489  3490  3491  3492  3493  3494  3495  3496  3497  3498  3499  generated =  [(-1, 77), (0, 1168), (1, 2255)]\n",
      "p_loss 1.559871 v_loss 0.279396\n",
      "vs_random =  [(-1, 3), (0, 4), (1, 93)] sum =  [(-1, 924), (0, 1268), (1, 15408)]\n",
      "game 3500  3501  3502  3503  3504  3505  3506  3507  3508  3509  3510  3511  3512  3513  3514  3515  3516  3517  3518  3519  generated =  [(-1, 78), (0, 1179), (1, 2263)]\n",
      "p_loss 1.602289 v_loss 0.272769\n",
      "vs_random =  [(-1, 4), (0, 1), (1, 95)] sum =  [(-1, 928), (0, 1269), (1, 15503)]\n",
      "game 3520  3521  3522  3523  3524  3525  3526  3527  3528  3529  3530  3531  3532  3533  3534  3535  3536  3537  3538  3539  generated =  [(-1, 79), (0, 1189), (1, 2272)]\n",
      "p_loss 1.507431 v_loss 0.268242\n",
      "vs_random =  [(-1, 7), (0, 10), (1, 83)] sum =  [(-1, 935), (0, 1279), (1, 15586)]\n",
      "game 3540  3541  3542  3543  3544  3545  3546  3547  3548  3549  3550  3551  3552  3553  3554  3555  3556  3557  3558  3559  generated =  [(-1, 79), (0, 1196), (1, 2285)]\n",
      "p_loss 1.585235 v_loss 0.264479\n",
      "vs_random =  [(-1, 6), (0, 3), (1, 91)] sum =  [(-1, 941), (0, 1282), (1, 15677)]\n",
      "game 3560  3561  3562  3563  3564  3565  3566  3567  3568  3569  3570  3571  3572  3573  3574  3575  3576  3577  3578  3579  generated =  [(-1, 79), (0, 1207), (1, 2294)]\n",
      "p_loss 1.561703 v_loss 0.256091\n",
      "vs_random =  [(-1, 3), (0, 8), (1, 89)] sum =  [(-1, 944), (0, 1290), (1, 15766)]\n",
      "game 3580  3581  3582  3583  3584  3585  3586  3587  3588  3589  3590  3591  3592  3593  3594  3595  3596  3597  3598  3599  generated =  [(-1, 79), (0, 1217), (1, 2304)]\n",
      "p_loss 1.530533 v_loss 0.267170\n",
      "vs_random =  [(-1, 8), (0, 8), (1, 84)] sum =  [(-1, 952), (0, 1298), (1, 15850)]\n",
      "game 3600  3601  3602  3603  3604  3605  3606  3607  3608  3609  3610  3611  3612  3613  3614  3615  3616  3617  3618  3619  generated =  [(-1, 79), (0, 1228), (1, 2313)]\n",
      "p_loss 1.556410 v_loss 0.280768\n",
      "vs_random =  [(-1, 4), (0, 9), (1, 87)] sum =  [(-1, 956), (0, 1307), (1, 15937)]\n",
      "game 3620  3621  3622  3623  3624  3625  3626  3627  3628  3629  3630  3631  3632  3633  3634  3635  3636  3637  3638  3639  generated =  [(-1, 79), (0, 1236), (1, 2325)]\n",
      "p_loss 1.542546 v_loss 0.268920\n",
      "vs_random =  [(-1, 3), (0, 6), (1, 91)] sum =  [(-1, 959), (0, 1313), (1, 16028)]\n",
      "game 3640  3641  3642  3643  3644  3645  3646  3647  3648  3649  3650  3651  3652  3653  3654  3655  3656  3657  3658  3659  generated =  [(-1, 79), (0, 1246), (1, 2335)]\n",
      "p_loss 1.611314 v_loss 0.276842\n",
      "vs_random =  [(0, 9), (1, 91)] sum =  [(-1, 959), (0, 1322), (1, 16119)]\n",
      "game 3660  3661  3662  3663  3664  3665  3666  3667  3668  3669  3670  3671  3672  3673  3674  3675  3676  3677  3678  3679  generated =  [(-1, 79), (0, 1253), (1, 2348)]\n",
      "p_loss 1.483315 v_loss 0.260253\n",
      "vs_random =  [(-1, 5), (0, 11), (1, 84)] sum =  [(-1, 964), (0, 1333), (1, 16203)]\n",
      "game 3680  3681  3682  3683  3684  3685  3686  3687  3688  3689  3690  3691  3692  3693  3694  3695  3696  3697  3698  3699  generated =  [(-1, 81), (0, 1260), (1, 2359)]\n",
      "p_loss 1.506840 v_loss 0.222598\n",
      "vs_random =  [(-1, 2), (0, 10), (1, 88)] sum =  [(-1, 966), (0, 1343), (1, 16291)]\n",
      "game 3700  3701  3702  3703  3704  3705  3706  3707  3708  3709  3710  3711  3712  3713  3714  3715  3716  3717  3718  3719  generated =  [(-1, 81), (0, 1266), (1, 2373)]\n",
      "p_loss 1.575056 v_loss 0.262824\n",
      "vs_random =  [(-1, 6), (0, 6), (1, 88)] sum =  [(-1, 972), (0, 1349), (1, 16379)]\n",
      "game 3720  3721  3722  3723  3724  3725  3726  3727  3728  3729  3730  3731  3732  3733  3734  3735  3736  3737  3738  3739  generated =  [(-1, 81), (0, 1275), (1, 2384)]\n",
      "p_loss 1.597327 v_loss 0.267206\n",
      "vs_random =  [(-1, 1), (0, 5), (1, 94)] sum =  [(-1, 973), (0, 1354), (1, 16473)]\n",
      "game 3740  3741  3742  3743  3744  3745  3746  3747  3748  3749  3750  3751  3752  3753  3754  3755  3756  3757  3758  3759  generated =  [(-1, 84), (0, 1281), (1, 2395)]\n",
      "p_loss 1.503635 v_loss 0.272386\n",
      "vs_random =  [(-1, 1), (0, 3), (1, 96)] sum =  [(-1, 974), (0, 1357), (1, 16569)]\n",
      "game 3760  3761  3762  3763  3764  3765  3766  3767  3768  3769  3770  3771  3772  3773  3774  3775  3776  3777  3778  3779  generated =  [(-1, 84), (0, 1290), (1, 2406)]\n",
      "p_loss 1.487425 v_loss 0.235733\n",
      "vs_random =  [(-1, 2), (0, 11), (1, 87)] sum =  [(-1, 976), (0, 1368), (1, 16656)]\n",
      "game 3780  3781  3782  3783  3784  3785  3786  3787  3788  3789  3790  3791  3792  3793  3794  3795  3796  3797  3798  3799  generated =  [(-1, 85), (0, 1297), (1, 2418)]\n",
      "p_loss 1.541765 v_loss 0.238717\n",
      "vs_random =  [(-1, 1), (0, 6), (1, 93)] sum =  [(-1, 977), (0, 1374), (1, 16749)]\n",
      "game 3800  3801  3802  3803  3804  3805  3806  3807  3808  3809  3810  3811  3812  3813  3814  3815  3816  3817  3818  3819  generated =  [(-1, 85), (0, 1307), (1, 2428)]\n",
      "p_loss 1.495473 v_loss 0.265575\n",
      "vs_random =  [(-1, 2), (0, 7), (1, 91)] sum =  [(-1, 979), (0, 1381), (1, 16840)]\n",
      "game 3820  3821  3822  3823  3824  3825  3826  3827  3828  3829  3830  3831  3832  3833  3834  3835  3836  3837  3838  3839  generated =  [(-1, 85), (0, 1317), (1, 2438)]\n",
      "p_loss 1.582000 v_loss 0.260670\n",
      "vs_random =  [(-1, 3), (0, 9), (1, 88)] sum =  [(-1, 982), (0, 1390), (1, 16928)]\n",
      "game 3840  3841  3842  3843  3844  3845  3846  3847  3848  3849  3850  3851  3852  3853  3854  3855  3856  3857  3858  3859  generated =  [(-1, 85), (0, 1327), (1, 2448)]\n",
      "p_loss 1.541090 v_loss 0.263777\n",
      "vs_random =  [(-1, 6), (0, 11), (1, 83)] sum =  [(-1, 988), (0, 1401), (1, 17011)]\n",
      "game 3860  3861  3862  3863  3864  3865  3866  3867  3868  3869  3870  3871  3872  3873  3874  3875  3876  3877  3878  3879  generated =  [(-1, 85), (0, 1337), (1, 2458)]\n",
      "p_loss 1.542359 v_loss 0.259201\n",
      "vs_random =  [(-1, 4), (0, 7), (1, 89)] sum =  [(-1, 992), (0, 1408), (1, 17100)]\n",
      "game 3880  3881  3882  3883  3884  3885  3886  3887  3888  3889  3890  3891  3892  3893  3894  3895  3896  3897  3898  3899  generated =  [(-1, 85), (0, 1347), (1, 2468)]\n",
      "p_loss 1.531828 v_loss 0.253590\n",
      "vs_random =  [(-1, 7), (0, 7), (1, 86)] sum =  [(-1, 999), (0, 1415), (1, 17186)]\n",
      "game 3900  3901  3902  3903  3904  3905  3906  3907  3908  3909  3910  3911  3912  3913  3914  3915  3916  3917  3918  3919  generated =  [(-1, 85), (0, 1356), (1, 2479)]\n",
      "p_loss 1.573672 v_loss 0.252455\n",
      "vs_random =  [(-1, 1), (0, 9), (1, 90)] sum =  [(-1, 1000), (0, 1424), (1, 17276)]\n",
      "game 3920  3921  3922  3923  3924  3925  3926  3927  3928  3929  3930  3931  3932  3933  3934  3935  3936  3937  3938  3939  generated =  [(-1, 85), (0, 1366), (1, 2489)]\n",
      "p_loss 1.526133 v_loss 0.285545\n",
      "vs_random =  [(-1, 3), (0, 5), (1, 92)] sum =  [(-1, 1003), (0, 1429), (1, 17368)]\n",
      "game 3940  3941  3942  3943  3944  3945  3946  3947  3948  3949  3950  3951  3952  3953  3954  3955  3956  3957  3958  3959  generated =  [(-1, 85), (0, 1377), (1, 2498)]\n",
      "p_loss 1.555937 v_loss 0.282645\n",
      "vs_random =  [(-1, 3), (0, 7), (1, 90)] sum =  [(-1, 1006), (0, 1436), (1, 17458)]\n",
      "game 3960  3961  3962  3963  3964  3965  3966  3967  3968  3969  3970  3971  3972  3973  3974  3975  3976  3977  3978  3979  generated =  [(-1, 85), (0, 1390), (1, 2505)]\n",
      "p_loss 1.533765 v_loss 0.263632\n",
      "vs_random =  [(-1, 2), (0, 8), (1, 90)] sum =  [(-1, 1008), (0, 1444), (1, 17548)]\n",
      "game 3980  3981  3982  3983  3984  3985  3986  3987  3988  3989  3990  3991  3992  3993  3994  3995  3996  3997  3998  3999  generated =  [(-1, 85), (0, 1401), (1, 2514)]\n",
      "p_loss 1.600490 v_loss 0.275304\n",
      "vs_random =  [(-1, 3), (0, 7), (1, 90)] sum =  [(-1, 1011), (0, 1451), (1, 17638)]\n",
      "game 4000  4001  4002  4003  4004  4005  4006  4007  4008  4009  4010  4011  4012  4013  4014  4015  4016  4017  4018  4019  generated =  [(-1, 85), (0, 1410), (1, 2525)]\n",
      "p_loss 1.558116 v_loss 0.257594\n",
      "vs_random =  [(-1, 1), (0, 4), (1, 95)] sum =  [(-1, 1012), (0, 1455), (1, 17733)]\n",
      "game 4020  4021  4022  4023  4024  4025  4026  4027  4028  4029  4030  4031  4032  4033  4034  4035  4036  4037  4038  4039  generated =  [(-1, 85), (0, 1423), (1, 2532)]\n",
      "p_loss 1.526277 v_loss 0.240979\n",
      "vs_random =  [(-1, 1), (0, 10), (1, 89)] sum =  [(-1, 1013), (0, 1465), (1, 17822)]\n",
      "game 4040  4041  4042  4043  4044  4045  4046  4047  4048  4049  4050  4051  4052  4053  4054  4055  4056  4057  4058  4059  generated =  [(-1, 85), (0, 1434), (1, 2541)]\n",
      "p_loss 1.532690 v_loss 0.274933\n",
      "vs_random =  [(-1, 4), (0, 1), (1, 95)] sum =  [(-1, 1017), (0, 1466), (1, 17917)]\n",
      "game 4060  4061  4062  4063  4064  4065  4066  4067  4068  4069  4070  4071  4072  4073  4074  4075  4076  4077  4078  4079  generated =  [(-1, 86), (0, 1442), (1, 2552)]\n",
      "p_loss 1.548635 v_loss 0.258493\n",
      "vs_random =  [(-1, 5), (0, 4), (1, 91)] sum =  [(-1, 1022), (0, 1470), (1, 18008)]\n",
      "game 4080  4081  4082  4083  4084  4085  4086  4087  4088  4089  4090  4091  4092  4093  4094  4095  4096  4097  4098  4099  generated =  [(-1, 86), (0, 1451), (1, 2563)]\n",
      "p_loss 1.522646 v_loss 0.254444\n",
      "vs_random =  [(-1, 2), (0, 7), (1, 91)] sum =  [(-1, 1024), (0, 1477), (1, 18099)]\n",
      "game 4100  4101  4102  4103  4104  4105  4106  4107  4108  4109  4110  4111  4112  4113  4114  4115  4116  4117  4118  4119  generated =  [(-1, 86), (0, 1460), (1, 2574)]\n",
      "p_loss 1.546982 v_loss 0.238886\n",
      "vs_random =  [(-1, 2), (0, 7), (1, 91)] sum =  [(-1, 1026), (0, 1484), (1, 18190)]\n",
      "game 4120  4121  4122  4123  4124  4125  4126  4127  4128  4129  4130  4131  4132  4133  4134  4135  4136  4137  4138  4139  generated =  [(-1, 86), (0, 1473), (1, 2581)]\n",
      "p_loss 1.456809 v_loss 0.230994\n",
      "vs_random =  [(-1, 2), (0, 8), (1, 90)] sum =  [(-1, 1028), (0, 1492), (1, 18280)]\n",
      "game 4140  4141  4142  4143  4144  4145  4146  4147  4148  4149  4150  4151  4152  4153  4154  4155  4156  4157  4158  4159  generated =  [(-1, 86), (0, 1482), (1, 2592)]\n",
      "p_loss 1.551619 v_loss 0.264321\n",
      "vs_random =  [(0, 6), (1, 94)] sum =  [(-1, 1028), (0, 1498), (1, 18374)]\n",
      "game 4160  4161  4162  4163  4164  4165  4166  4167  4168  4169  4170  4171  4172  4173  4174  4175  4176  4177  4178  4179  generated =  [(-1, 86), (0, 1495), (1, 2599)]\n",
      "p_loss 1.529405 v_loss 0.251447\n",
      "vs_random =  [(-1, 2), (0, 3), (1, 95)] sum =  [(-1, 1030), (0, 1501), (1, 18469)]\n",
      "game 4180  4181  4182  4183  4184  4185  4186  4187  4188  4189  4190  4191  4192  4193  4194  4195  4196  4197  4198  4199  generated =  [(-1, 87), (0, 1502), (1, 2611)]\n",
      "p_loss 1.505935 v_loss 0.251468\n",
      "vs_random =  [(-1, 2), (0, 5), (1, 93)] sum =  [(-1, 1032), (0, 1506), (1, 18562)]\n",
      "game 4200  4201  4202  4203  4204  4205  4206  4207  4208  4209  4210  4211  4212  4213  4214  4215  4216  4217  4218  4219  generated =  [(-1, 89), (0, 1512), (1, 2619)]\n",
      "p_loss 1.539969 v_loss 0.251866\n",
      "vs_random =  [(-1, 1), (0, 8), (1, 91)] sum =  [(-1, 1033), (0, 1514), (1, 18653)]\n",
      "game 4220  4221  4222  4223  4224  4225  4226  4227  4228  4229  4230  4231  4232  4233  4234  4235  4236  4237  4238  4239  generated =  [(-1, 89), (0, 1523), (1, 2628)]\n",
      "p_loss 1.544433 v_loss 0.235298\n",
      "vs_random =  [(-1, 7), (0, 6), (1, 87)] sum =  [(-1, 1040), (0, 1520), (1, 18740)]\n",
      "game 4240  4241  4242  4243  4244  4245  4246  4247  4248  4249  4250  4251  4252  4253  4254  4255  4256  4257  4258  4259  generated =  [(-1, 89), (0, 1535), (1, 2636)]\n",
      "p_loss 1.616767 v_loss 0.285133\n",
      "vs_random =  [(-1, 3), (0, 2), (1, 95)] sum =  [(-1, 1043), (0, 1522), (1, 18835)]\n",
      "game 4260  4261  4262  4263  4264  4265  4266  4267  4268  4269  4270  4271  4272  4273  4274  4275  4276  4277  4278  4279  generated =  [(-1, 89), (0, 1547), (1, 2644)]\n",
      "p_loss 1.489452 v_loss 0.255935\n",
      "vs_random =  [(-1, 5), (0, 7), (1, 88)] sum =  [(-1, 1048), (0, 1529), (1, 18923)]\n",
      "game 4280  4281  4282  4283  4284  4285  4286  4287  4288  4289  4290  4291  4292  4293  4294  4295  4296  4297  4298  4299  generated =  [(-1, 89), (0, 1554), (1, 2657)]\n",
      "p_loss 1.541813 v_loss 0.241847\n",
      "vs_random =  [(-1, 5), (0, 7), (1, 88)] sum =  [(-1, 1053), (0, 1536), (1, 19011)]\n",
      "game 4300  4301  4302  4303  4304  4305  4306  4307  4308  4309  4310  4311  4312  4313  4314  4315  4316  4317  4318  4319  generated =  [(-1, 89), (0, 1565), (1, 2666)]\n",
      "p_loss 1.480895 v_loss 0.256285\n",
      "vs_random =  [(-1, 6), (0, 11), (1, 83)] sum =  [(-1, 1059), (0, 1547), (1, 19094)]\n",
      "game 4320  4321  4322  4323  4324  4325  4326  4327  4328  4329  4330  4331  4332  4333  4334  4335  4336  4337  4338  4339  generated =  [(-1, 89), (0, 1579), (1, 2672)]\n",
      "p_loss 1.493775 v_loss 0.233617\n",
      "vs_random =  [(-1, 8), (0, 9), (1, 83)] sum =  [(-1, 1067), (0, 1556), (1, 19177)]\n",
      "game 4340  4341  4342  4343  4344  4345  4346  4347  4348  4349  4350  4351  4352  4353  4354  4355  4356  4357  4358  4359  generated =  [(-1, 89), (0, 1592), (1, 2679)]\n",
      "p_loss 1.482255 v_loss 0.255164\n",
      "vs_random =  [(-1, 3), (0, 12), (1, 85)] sum =  [(-1, 1070), (0, 1568), (1, 19262)]\n",
      "game 4360  4361  4362  4363  4364  4365  4366  4367  4368  4369  4370  4371  4372  4373  4374  4375  4376  4377  4378  4379  generated =  [(-1, 89), (0, 1605), (1, 2686)]\n",
      "p_loss 1.547273 v_loss 0.261548\n",
      "vs_random =  [(-1, 5), (0, 11), (1, 84)] sum =  [(-1, 1075), (0, 1579), (1, 19346)]\n",
      "game 4380  4381  4382  4383  4384  4385  4386  4387  4388  4389  4390  4391  4392  4393  4394  4395  4396  4397  4398  4399  generated =  [(-1, 89), (0, 1617), (1, 2694)]\n",
      "p_loss 1.501623 v_loss 0.250211\n",
      "vs_random =  [(-1, 7), (0, 8), (1, 85)] sum =  [(-1, 1082), (0, 1587), (1, 19431)]\n",
      "game 4400  4401  4402  4403  4404  4405  4406  4407  4408  4409  4410  4411  4412  4413  4414  4415  4416  4417  4418  4419  generated =  [(-1, 89), (0, 1632), (1, 2699)]\n",
      "p_loss 1.462213 v_loss 0.254564\n",
      "vs_random =  [(-1, 2), (0, 12), (1, 86)] sum =  [(-1, 1084), (0, 1599), (1, 19517)]\n",
      "game 4420  4421  4422  4423  4424  4425  4426  4427  4428  4429  4430  4431  4432  4433  4434  4435  4436  4437  4438  4439  generated =  [(-1, 89), (0, 1642), (1, 2709)]\n",
      "p_loss 1.551204 v_loss 0.233009\n",
      "vs_random =  [(-1, 3), (0, 7), (1, 90)] sum =  [(-1, 1087), (0, 1606), (1, 19607)]\n",
      "game 4440  4441  4442  4443  4444  4445  4446  4447  4448  4449  4450  4451  4452  4453  4454  4455  4456  4457  4458  4459  generated =  [(-1, 90), (0, 1650), (1, 2720)]\n",
      "p_loss 1.486123 v_loss 0.247453\n",
      "vs_random =  [(-1, 9), (0, 9), (1, 82)] sum =  [(-1, 1096), (0, 1615), (1, 19689)]\n",
      "game 4460  4461  4462  4463  4464  4465  4466  4467  4468  4469  4470  4471  4472  4473  4474  4475  4476  4477  4478  4479  generated =  [(-1, 90), (0, 1660), (1, 2730)]\n",
      "p_loss 1.521322 v_loss 0.247916\n",
      "vs_random =  [(-1, 3), (0, 5), (1, 92)] sum =  [(-1, 1099), (0, 1620), (1, 19781)]\n",
      "game 4480  4481  4482  4483  4484  4485  4486  4487  4488  4489  4490  4491  4492  4493  4494  4495  4496  4497  4498  4499  generated =  [(-1, 90), (0, 1673), (1, 2737)]\n",
      "p_loss 1.506403 v_loss 0.254123\n",
      "vs_random =  [(-1, 2), (0, 10), (1, 88)] sum =  [(-1, 1101), (0, 1630), (1, 19869)]\n",
      "game 4500  4501  4502  4503  4504  4505  4506  4507  4508  4509  4510  4511  4512  4513  4514  4515  4516  4517  4518  4519  generated =  [(-1, 90), (0, 1682), (1, 2748)]\n",
      "p_loss 1.551061 v_loss 0.253744\n",
      "vs_random =  [(-1, 1), (0, 9), (1, 90)] sum =  [(-1, 1102), (0, 1639), (1, 19959)]\n",
      "game 4520  4521  4522  4523  4524  4525  4526  4527  4528  4529  4530  4531  4532  4533  4534  4535  4536  4537  4538  4539  generated =  [(-1, 90), (0, 1697), (1, 2753)]\n",
      "p_loss 1.531456 v_loss 0.232976\n",
      "vs_random =  [(-1, 6), (0, 11), (1, 83)] sum =  [(-1, 1108), (0, 1650), (1, 20042)]\n",
      "game 4540  4541  4542  4543  4544  4545  4546  4547  4548  4549  4550  4551  4552  4553  4554  4555  4556  4557  4558  4559  generated =  [(-1, 90), (0, 1709), (1, 2761)]\n",
      "p_loss 1.534556 v_loss 0.256582\n",
      "vs_random =  [(-1, 7), (0, 9), (1, 84)] sum =  [(-1, 1115), (0, 1659), (1, 20126)]\n",
      "game 4560  4561  4562  4563  4564  4565  4566  4567  4568  4569  4570  4571  4572  4573  4574  4575  4576  4577  4578  4579  generated =  [(-1, 90), (0, 1719), (1, 2771)]\n",
      "p_loss 1.493136 v_loss 0.239000\n",
      "vs_random =  [(-1, 4), (0, 7), (1, 89)] sum =  [(-1, 1119), (0, 1666), (1, 20215)]\n",
      "game 4580  4581  4582  4583  4584  4585  4586  4587  4588  4589  4590  4591  4592  4593  4594  4595  4596  4597  4598  4599  generated =  [(-1, 90), (0, 1729), (1, 2781)]\n",
      "p_loss 1.518074 v_loss 0.254274\n",
      "vs_random =  [(-1, 5), (0, 9), (1, 86)] sum =  [(-1, 1124), (0, 1675), (1, 20301)]\n",
      "game 4600  4601  4602  4603  4604  4605  4606  4607  4608  4609  4610  4611  4612  4613  4614  4615  4616  4617  4618  4619  generated =  [(-1, 90), (0, 1743), (1, 2787)]\n",
      "p_loss 1.484677 v_loss 0.226943\n",
      "vs_random =  [(-1, 2), (0, 6), (1, 92)] sum =  [(-1, 1126), (0, 1681), (1, 20393)]\n",
      "game 4620  4621  4622  4623  4624  4625  4626  4627  4628  4629  4630  4631  4632  4633  4634  4635  4636  4637  4638  4639  generated =  [(-1, 90), (0, 1753), (1, 2797)]\n",
      "p_loss 1.520922 v_loss 0.234332\n",
      "vs_random =  [(-1, 4), (0, 11), (1, 85)] sum =  [(-1, 1130), (0, 1692), (1, 20478)]\n",
      "game 4640  4641  4642  4643  4644  4645  4646  4647  4648  4649  4650  4651  4652  4653  4654  4655  4656  4657  4658  4659  generated =  [(-1, 90), (0, 1765), (1, 2805)]\n",
      "p_loss 1.492869 v_loss 0.227025\n",
      "vs_random =  [(-1, 3), (0, 11), (1, 86)] sum =  [(-1, 1133), (0, 1703), (1, 20564)]\n",
      "game 4660  4661  4662  4663  4664  4665  4666  4667  4668  4669  4670  4671  4672  4673  4674  4675  4676  4677  4678  4679  generated =  [(-1, 90), (0, 1778), (1, 2812)]\n",
      "p_loss 1.496334 v_loss 0.233895\n",
      "vs_random =  [(-1, 2), (0, 3), (1, 95)] sum =  [(-1, 1135), (0, 1706), (1, 20659)]\n",
      "game 4680  4681  4682  4683  4684  4685  4686  4687  4688  4689  4690  4691  4692  4693  4694  4695  4696  4697  4698  4699  generated =  [(-1, 91), (0, 1795), (1, 2814)]\n",
      "p_loss 1.550182 v_loss 0.273692\n",
      "vs_random =  [(-1, 5), (0, 14), (1, 81)] sum =  [(-1, 1140), (0, 1720), (1, 20740)]\n",
      "game 4700  4701  4702  4703  4704  4705  4706  4707  4708  4709  4710  4711  4712  4713  4714  4715  4716  4717  4718  4719  generated =  [(-1, 91), (0, 1810), (1, 2819)]\n",
      "p_loss 1.509197 v_loss 0.238405\n",
      "vs_random =  [(-1, 5), (0, 8), (1, 87)] sum =  [(-1, 1145), (0, 1728), (1, 20827)]\n",
      "game 4720  4721  4722  4723  4724  4725  4726  4727  4728  4729  4730  4731  4732  4733  4734  4735  4736  4737  4738  4739  generated =  [(-1, 91), (0, 1821), (1, 2828)]\n",
      "p_loss 1.426878 v_loss 0.228446\n",
      "vs_random =  [(-1, 4), (0, 8), (1, 88)] sum =  [(-1, 1149), (0, 1736), (1, 20915)]\n",
      "game 4740  4741  4742  4743  4744  4745  4746  4747  4748  4749  4750  4751  4752  4753  4754  4755  4756  4757  4758  4759  generated =  [(-1, 91), (0, 1836), (1, 2833)]\n",
      "p_loss 1.518155 v_loss 0.271227\n",
      "vs_random =  [(-1, 2), (0, 12), (1, 86)] sum =  [(-1, 1151), (0, 1748), (1, 21001)]\n",
      "game 4760  4761  4762  4763  4764  4765  4766  4767  4768  4769  4770  4771  4772  4773  4774  4775  4776  4777  4778  4779  generated =  [(-1, 92), (0, 1849), (1, 2839)]\n",
      "p_loss 1.484389 v_loss 0.252907\n",
      "vs_random =  [(-1, 2), (0, 8), (1, 90)] sum =  [(-1, 1153), (0, 1756), (1, 21091)]\n",
      "game 4780  4781  4782  4783  4784  4785  4786  4787  4788  4789  4790  4791  4792  4793  4794  4795  4796  4797  4798  4799  generated =  [(-1, 92), (0, 1857), (1, 2851)]\n",
      "p_loss 1.476917 v_loss 0.232106\n",
      "vs_random =  [(0, 10), (1, 90)] sum =  [(-1, 1153), (0, 1766), (1, 21181)]\n",
      "game 4800  4801  4802  4803  4804  4805  4806  4807  4808  4809  4810  4811  4812  4813  4814  4815  4816  4817  4818  4819  generated =  [(-1, 92), (0, 1871), (1, 2857)]\n",
      "p_loss 1.452780 v_loss 0.253040\n",
      "vs_random =  [(-1, 7), (0, 10), (1, 83)] sum =  [(-1, 1160), (0, 1776), (1, 21264)]\n",
      "game 4820  4821  4822  4823  4824  4825  4826  4827  4828  4829  4830  4831  4832  4833  4834  4835  4836  4837  4838  4839  generated =  [(-1, 92), (0, 1883), (1, 2865)]\n",
      "p_loss 1.457294 v_loss 0.232793\n",
      "vs_random =  [(-1, 5), (0, 7), (1, 88)] sum =  [(-1, 1165), (0, 1783), (1, 21352)]\n",
      "game 4840  4841  4842  4843  4844  4845  4846  4847  4848  4849  4850  4851  4852  4853  4854  4855  4856  4857  4858  4859  generated =  [(-1, 92), (0, 1896), (1, 2872)]\n",
      "p_loss 1.525706 v_loss 0.265263\n",
      "vs_random =  [(-1, 5), (0, 5), (1, 90)] sum =  [(-1, 1170), (0, 1788), (1, 21442)]\n",
      "game 4860  4861  4862  4863  4864  4865  4866  4867  4868  4869  4870  4871  4872  4873  4874  4875  4876  4877  4878  4879  generated =  [(-1, 92), (0, 1906), (1, 2882)]\n",
      "p_loss 1.507472 v_loss 0.251595\n",
      "vs_random =  [(-1, 4), (0, 8), (1, 88)] sum =  [(-1, 1174), (0, 1796), (1, 21530)]\n",
      "game 4880  4881  4882  4883  4884  4885  4886  4887  4888  4889  4890  4891  4892  4893  4894  4895  4896  4897  4898  4899  generated =  [(-1, 92), (0, 1924), (1, 2884)]\n",
      "p_loss 1.518506 v_loss 0.261009\n",
      "vs_random =  [(-1, 5), (0, 7), (1, 88)] sum =  [(-1, 1179), (0, 1803), (1, 21618)]\n",
      "game 4900  4901  4902  4903  4904  4905  4906  4907  4908  4909  4910  4911  4912  4913  4914  4915  4916  4917  4918  4919  generated =  [(-1, 92), (0, 1938), (1, 2890)]\n",
      "p_loss 1.477079 v_loss 0.252598\n",
      "vs_random =  [(-1, 3), (0, 5), (1, 92)] sum =  [(-1, 1182), (0, 1808), (1, 21710)]\n",
      "game 4920  4921  4922  4923  4924  4925  4926  4927  4928  4929  4930  4931  4932  4933  4934  4935  4936  4937  4938  4939  generated =  [(-1, 92), (0, 1952), (1, 2896)]\n",
      "p_loss 1.457805 v_loss 0.241476\n",
      "vs_random =  [(-1, 6), (0, 7), (1, 87)] sum =  [(-1, 1188), (0, 1815), (1, 21797)]\n",
      "game 4940  4941  4942  4943  4944  4945  4946  4947  4948  4949  4950  4951  4952  4953  4954  4955  4956  4957  4958  4959  generated =  [(-1, 93), (0, 1962), (1, 2905)]\n",
      "p_loss 1.474001 v_loss 0.238818\n",
      "vs_random =  [(-1, 3), (0, 12), (1, 85)] sum =  [(-1, 1191), (0, 1827), (1, 21882)]\n",
      "game 4960  4961  4962  4963  4964  4965  4966  4967  4968  4969  4970  4971  4972  4973  4974  4975  4976  4977  4978  4979  generated =  [(-1, 93), (0, 1977), (1, 2910)]\n",
      "p_loss 1.485373 v_loss 0.249757\n",
      "vs_random =  [(-1, 4), (0, 4), (1, 92)] sum =  [(-1, 1195), (0, 1831), (1, 21974)]\n",
      "game 4980  4981  4982  4983  4984  4985  4986  4987  4988  4989  4990  4991  4992  4993  4994  4995  4996  4997  4998  4999  generated =  [(-1, 93), (0, 1990), (1, 2917)]\n",
      "p_loss 1.488365 v_loss 0.235353\n",
      "vs_random =  [(-1, 3), (0, 6), (1, 91)] sum =  [(-1, 1198), (0, 1837), (1, 22065)]\n",
      "finished\n"
     ]
    }
   ],
   "source": [
    "# Main algorithm of MuZero\r\n",
    "\r\n",
    "num_games = 5000 #  默认为50000\r\n",
    "num_games_one_epoch = 20\r\n",
    "num_simulations = 40\r\n",
    "\r\n",
    "net = Net()\r\n",
    "####参数读取\r\n",
    "model_state_dict = paddle.load(\"net0.pdparams\")\r\n",
    "net.set_state_dict(model_state_dict)\r\n",
    "\r\n",
    "optimizer = optim.Momentum(parameters=net.parameters(), learning_rate=3e-4, weight_decay=3e-5,momentum=0.8)\r\n",
    "\r\n",
    "# Display battle results as {-1: lose 0: draw 1: win} (for episode generated for training, 1 means that the first player won)\r\n",
    "vs_random_sum = vs_random(net)\r\n",
    "print('vs_random = ', sorted(vs_random_sum.items()))\r\n",
    "\r\n",
    "episodes = []\r\n",
    "result_distribution = {1: 0, 0: 0, -1: 0}\r\n",
    "\r\n",
    "for g in range(num_games):\r\n",
    "    # Generate one episode\r\n",
    "    record, p_targets, features, action_features = [], [], [], []\r\n",
    "    state = State()\r\n",
    "    # temperature using to make policy targets from search results\r\n",
    "    temperature = 0.7\r\n",
    "    while not state.terminal():\r\n",
    "        tree = Tree(net)\r\n",
    "        p_target = tree.think(state, num_simulations, temperature)\r\n",
    "        p_targets.append(p_target)\r\n",
    "        features.append(state.feature())\r\n",
    "        # Select action with generated distribution, and then make a transition by that action\r\n",
    "        action = np.random.choice(np.arange(len(p_target)), p=p_target)\r\n",
    "        action_features.append(state.action_feature(action))\r\n",
    "        state.play(action)\r\n",
    "        record.append(action)\r\n",
    "        temperature *= 0.8\r\n",
    "    # reward seen from the first turn player\r\n",
    "    reward = state.terminal_reward() * (1 if len(record) % 2 == 0 else -1)\r\n",
    "    result_distribution[reward] += 1\r\n",
    "    episodes.append((record, reward, features, action_features, p_targets))\r\n",
    "    if g % num_games_one_epoch == 0:\r\n",
    "        print('game ', end='')\r\n",
    "    print(g, ' ', end='')\r\n",
    "\r\n",
    "    # Training of neural net\r\n",
    "    if (g + 1) % num_games_one_epoch == 0:\r\n",
    "        # Show the result distributiuon of generated episodes\r\n",
    "        print('generated = ', sorted(result_distribution.items()))\r\n",
    "        net = train(episodes, net, optimizer)\r\n",
    "        vs_random_once = vs_random(net)\r\n",
    "        print('vs_random = ', sorted(vs_random_once.items()), end='')\r\n",
    "        for r, n in vs_random_once.items():\r\n",
    "            vs_random_sum[r] += n\r\n",
    "        print(' sum = ', sorted(vs_random_sum.items()))\r\n",
    "        #show_net(net, State())\r\n",
    "        #show_net(net, State().play('A1 C1 A2 C2'))\r\n",
    "        #show_net(net, State().play('A1 B2 C3 B3 C1'))\r\n",
    "        #show_net(net, State().play('B2 A2 A3 C1 B3'))\r\n",
    "        #show_net(net, State().play('B2 A2 A3 C1'))\r\n",
    "print('finished')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "###根据时间戳保存参数\r\n",
    "import time\r\n",
    "t =time.time()\r\n",
    "model_state_dict = net.state_dict()\r\n",
    "paddle.save(model_state_dict, \"net\"+str(int(t))+\".pdparams\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PaddlePaddle 2.0.0b0 (Python 3.5)",
   "language": "python",
   "name": "py35-paddle1.2.0"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}