RL_for_connect4_training.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "RL_for_connect4.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KcE2RqAvbbjC"
      },
      "source": [
        "## Mounting Google drive to save and retrive the model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IGGlil13Do_m",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1675cb7d-94b5-4c82-eb0e-d8de372fea4e"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "977Ph7AYboyI"
      },
      "source": [
        "# Definition of All Required Classes"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dCKJZ7s3b3y6"
      },
      "source": [
        "### Importing Libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ws2Obf5Vm-y0"
      },
      "source": [
        "import math\n",
        "%tensorflow_version 2.x\n",
        "import numpy as np\n",
        "from collections import deque\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras import Model\n",
        "from tensorflow.keras.layers import Conv2D, Dense, Flatten, Activation, Dropout, MaxPooling2D, Input, Add, BatchNormalization, LeakyReLU\n",
        "from tensorflow.keras.models import load_model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ccE-eU0ab_1p"
      },
      "source": [
        "### Game class, which has methods and rules to play connect 4"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gmSF4qf3nG4O"
      },
      "source": [
        "import numpy as np\n",
        "from copy import deepcopy\n",
        "\n",
        "class Game():\n",
        "    def __init__(self,state=np.zeros((6,7)),player=1):\n",
        "        self.state = deepcopy(state)\n",
        "        self.player = player\n",
        "        #self.top = self.gettop(state)        \n",
        "        pass\n",
        "    \n",
        "    def gettop(self,state):\n",
        "        top = np.zeros((7,),dtype=np.int32)\n",
        "        for i in range(7):\n",
        "            for j in range(6):\n",
        "                if state[j,i]==0:\n",
        "                    top[i]=j\n",
        "                    break\n",
        "                top[i]=6\n",
        "        return top\n",
        "    \n",
        "    def ValidMoves(self,state,player):\n",
        "        top=self.gettop(state)\n",
        "        return top<6    \n",
        "    \n",
        "    def Next(self,state,player,action):\n",
        "        state=deepcopy(state)\n",
        "        for j in range(6):\n",
        "            if state[j,action]==0:\n",
        "                state[j,action]=player\n",
        "                break\n",
        "        return state, -player\n",
        "    \n",
        "    def play(self,action):\n",
        "        self.state[self.gettop(self.state)[action],action]=self.player\n",
        "        self.player*=-1\n",
        "        pass\n",
        "    \n",
        "    def end(self,state,player):\n",
        "        tok=player\n",
        "        for i in range(4):\n",
        "            for j in range(6):\n",
        "                if state[j][i]==tok and state[j][i+1]==tok and state[j][i+2]==tok and state[j][i+3]==tok:\n",
        "                    return 1\n",
        "\n",
        "            for j in range(2):\n",
        "                if state[j][i]==tok and state[j+1][i+1]==tok and state[j+2][i+2]==tok and state[j+3][i+3]==tok:\n",
        "                    return 1\n",
        "\n",
        "            for j in range(3,6):\n",
        "                if state[j][i]==tok and state[j-1][i+1]==tok and state[j-2][i+2]==tok and state[j-3][i+3]==tok:\n",
        "                    return 1\n",
        "\n",
        "        for i in range(7):\n",
        "            for j in range(3):\n",
        "                if state[j][i]==tok and state[j+1][i]==tok and state[j+2][i]==tok and state[j+3][i]==tok:\n",
        "                    return 1\n",
        "\n",
        "        tok=-player\n",
        "        for i in range(4):\n",
        "            for j in range(6):\n",
        "                if state[j][i]==tok and state[j][i+1]==tok and state[j][i+2]==tok and state[j][i+3]==tok:\n",
        "                    return -1\n",
        "\n",
        "            for j in range(2):\n",
        "                if state[j][i]==tok and state[j+1][i+1]==tok and state[j+2][i+2]==tok and state[j+3][i+3]==tok:\n",
        "                    return -1\n",
        "\n",
        "            for j in range(3,6):\n",
        "                if state[j][i]==tok and state[j-1][i+1]==tok and state[j-2][i+2]==tok and state[j-3][i+3]==tok:\n",
        "                    return -1   \n",
        "        for i in range(7):\n",
        "            for j in range(3):\n",
        "                if state[j][i]==tok and state[j+1][i]==tok and state[j+2][i]==tok and state[j+3][i]==tok:\n",
        "                    return -1\n",
        "\n",
        "        if min(self.gettop(state))==6:\n",
        "            return 0\n",
        "        return -2\n",
        "    \n",
        "    def fstategen(self,state,player):\n",
        "        return (state*player)\n",
        "    \n",
        "    def stringrep(self,state):\n",
        "        return state.tobytes()\n",
        "            \n",
        "    def PrintBoard(self,state):\n",
        "        for i in range(5,-1,-1):\n",
        "            for j in range(7):\n",
        "                if state[i,j]==1:\n",
        "                    print(\"|X\", end=\"\")\n",
        "                elif state[i,j]==-1:\n",
        "                    print(\"|O\", end=\"\")\n",
        "                else:\n",
        "                    print(\"| \", end=\"\")\n",
        "            print(\"|\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_kZwcGl0cODp"
      },
      "source": [
        "### Network class contains neural network and it's required methods"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p8ydeGhonQDh"
      },
      "source": [
        "#%tensorflow_version 2.x\n",
        "import numpy as np\n",
        "import math\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras import Model\n",
        "from tensorflow.keras.layers import Conv2D, Dense, Flatten, Activation, Dropout, MaxPooling2D, Input, Add\n",
        "\n",
        "class Network():\n",
        "    def generate_model(self):\n",
        "        def identity_block(X, f, filters):\n",
        "            F1, F2 = filters\n",
        "            \n",
        "            X_shortcut = X\n",
        "            \n",
        "            X = Conv2D(filters = F1, kernel_size = (1, 1), strides = (1,1))(X)\n",
        "            X = LeakyReLU()(X)\n",
        "                \n",
        "            X = Conv2D(filters = F2, kernel_size = (f, f), strides = (1,1), padding = 'same')(X)\n",
        "            X = BatchNormalization()(X)\n",
        "            X = Add()([X_shortcut, X])\n",
        "            X = LeakyReLU()(X)\n",
        "            return X\n",
        "        input = Input(shape=(6,7,2))\n",
        "\n",
        "        layer = Conv2D(filters = 8, kernel_size = (1, 1), strides = (1,1))(input)\n",
        "        layer=LeakyReLU()(layer)\n",
        "        layer=identity_block(layer, 4, [8, 8])\n",
        "        layer=identity_block(layer, 3, [8, 8])\n",
        "        layer=MaxPooling2D(pool_size=(3,3),strides=(1,1))(layer)\n",
        "\n",
        "        layer = Conv2D(filters = 16, kernel_size = (1, 1), strides = (1,1))(layer)\n",
        "        layer=LeakyReLU()(layer)\n",
        "        layer=identity_block(layer, 3, [16, 16])\n",
        "        layer=identity_block(layer, 2, [16, 16])\n",
        "        layer=MaxPooling2D(pool_size=(2,2),strides=(1,1))(layer)\n",
        "        layer = Conv2D(filters = 32, kernel_size = (3, 3), strides = (1,1))(layer)\n",
        "        layer=LeakyReLU()(layer)\n",
        "\n",
        "        layer=Flatten()(layer)\n",
        "\n",
        "        #--------VALUE NETWORK---------\n",
        "        valuehead=Dense(32)(layer)\n",
        "        layer=LeakyReLU()(layer)\n",
        "        valuehead = Dropout(0.4)(valuehead)\n",
        "\n",
        "        valuehead=Dense(8)(valuehead)\n",
        "        valuehead=LeakyReLU()(valuehead)\n",
        "        valuehead = Dropout(0.2)(valuehead)\n",
        "        \n",
        "        valuehead=Dense(4)(valuehead)\n",
        "        valuehead=LeakyReLU()(valuehead)\n",
        "\n",
        "        valuehead=Dense(1, activation='tanh', name=\"valuehead\")(valuehead)\n",
        "\n",
        "\n",
        "        #--------POLICY NETWORK---------\n",
        "        policyhead=Dense(56)(layer)\n",
        "        policyhead=LeakyReLU()(policyhead)\n",
        "        policyhead = Dropout(0.3)(policyhead)\n",
        "\n",
        "        policyhead=Dense(42)(policyhead)\n",
        "        policyhead=LeakyReLU()(policyhead)\n",
        "        policyhead = Dropout(0.1)(policyhead)\n",
        "\n",
        "        policyhead=Dense(14)(policyhead)\n",
        "        policyhead=LeakyReLU()(policyhead)\n",
        "\n",
        "        policyhead=Dense(7, activation='softmax',name=\"policyhead\")(policyhead)\n",
        "\n",
        "        model = Model(inputs=input, outputs=[valuehead ,policyhead])\n",
        "        opt = tf.keras.optimizers.SGD(momentum=0.1)\n",
        "        loss1 = tf.keras.losses.MeanSquaredError()\n",
        "        loss2 = self.customLoss\n",
        "        losses = {\n",
        "            \"valuehead\": loss1,\n",
        "            \"policyhead\": loss2,\n",
        "        }\n",
        "        model.compile(loss=losses, optimizer=opt)\n",
        "        return model\n",
        "\n",
        "    def __init__(self, name=None):\n",
        "        if name==None:\n",
        "            self.model=self.generate_model()\n",
        "            self.champion=self.generate_model()\n",
        "        else:\n",
        "            self.model = load_model(name, custom_objects={self.customLoss.__name__: self.customLoss})\n",
        "            self.champion = load_model(name, custom_objects={self.customLoss.__name__: self.customLoss})\n",
        "            \n",
        "    def champ_predict(self, state, p):\n",
        "        st=np.reshape(state, (1,state.shape[0],state.shape[1],-1) )\n",
        "        inp1=(st*p>0).astype(np.float32)\n",
        "        inp2=(st*p<0).astype(np.float32)\n",
        "        inp=np.concatenate( (inp1,inp2 ), axis=3)\n",
        "        return self.champion.predict(inp)\n",
        "\n",
        "    def predict(self, state, p):\n",
        "        st=np.reshape(state, (1,state.shape[0],state.shape[1],-1) )\n",
        "        inp1=(st*p>0).astype(np.float32)\n",
        "        inp2=(st*p<0).astype(np.float32)\n",
        "        inp=np.concatenate( (inp1,inp2 ), axis=3)\n",
        "        return self.model.predict(inp)\n",
        "    \n",
        "    def still_champ(self,name=\"check\"):\n",
        "        self.champion.save(name)\n",
        "        tf.keras.backend.clear_session()\n",
        "        self.model = load_model(name, custom_objects={self.customLoss.__name__: self.customLoss})\n",
        "        self.champion = load_model(name, custom_objects={self.customLoss.__name__: self.customLoss})\n",
        "    \n",
        "    def new_champ(self,name=\"check\"):\n",
        "        self.model.save(name)\n",
        "        tf.keras.backend.clear_session()        \n",
        "        self.model = load_model(name, custom_objects={self.customLoss.__name__: self.customLoss})\n",
        "        self.champion = load_model(name, custom_objects={self.customLoss.__name__: self.customLoss})\n",
        "    \n",
        "    @tf.function\n",
        "    def customLoss(self,y_true, y_pred):\n",
        "        #print(y_true)\n",
        "        #print(y_pred)\n",
        "        loss =tf.reduce_mean(tf.reduce_sum(-tf.math.multiply(y_true, tf.math.log(y_pred)), axis=1, keepdims=True) )\n",
        "        return loss\n",
        "\n",
        "    def preprocess(self,X,Y1,Y2):\n",
        "        X=np.array(X)\n",
        "        Y1=np.array(Y1).astype(np.float32)\n",
        "        Y2=np.array(Y2).astype(np.float32)\n",
        "        X=np.reshape(X,(X.shape[0],X.shape[1],X.shape[2],-1))\n",
        "        inp1=(X>0).astype(np.float32)\n",
        "        inp2=(X<0).astype(np.float32)\n",
        "        X=np.concatenate( (inp1,inp2 ), axis=3)\n",
        "        indices = np.arange(X.shape[0])\n",
        "        np.random.shuffle(indices)\n",
        "        X=X[indices]\n",
        "        Y1 = Y1[indices]\n",
        "        Y2 = Y2[indices]\n",
        "        return X,Y1,Y2\n",
        "\n",
        "    def train(self, X,Y1,Y2,Epochs=3, Batch=64):        \n",
        "        X,Y1,Y2=self.preprocess(X, Y1, Y2)\n",
        "        Y= {\"valuehead\": Y1,\n",
        "            \"policyhead\": Y2 }\n",
        "        #print(X.shape)\n",
        "        #print(X[5,:,:,0])\n",
        "        #print(X[5,:,:,1])\n",
        "        #print(Y)\n",
        "        self.model.fit(X, Y, batch_size=Batch, epochs=Epochs)\n",
        "        pass"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7wZ8pxpgcYMp"
      },
      "source": [
        "### Monte Carlo tree search, Used to find best move."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kbt3Cpb1nLH-"
      },
      "source": [
        "class Mcts():\n",
        "    def __init__(self,game,nn,cput,num):\n",
        "        self.game = game\n",
        "        self.nn = nn\n",
        "        self.cput = cput\n",
        "        self.num = num\n",
        "        self.Qsa = {}\n",
        "        self.Nsa = {}\n",
        "        self.Ns = {}\n",
        "        self.Ps = {}\n",
        "        self.Es = {}\n",
        "        self.Vs = {}\n",
        "    \n",
        "    def search(self,fstate):\n",
        "        s = fstate.tobytes()\n",
        "        \n",
        "        if s not in self.Es:\n",
        "            self.Es[s] = self.game.end(fstate,1)\n",
        "        if self.Es[s]!=-2:\n",
        "            return -self.Es[s]\n",
        "        \n",
        "        if s not in self.Ps:\n",
        "            [v,tempP] = self.nn.predict(fstate,1)\n",
        "            v = v[0][0]\n",
        "            self.Ps[s] = tempP[0]\n",
        "            valids = self.game.ValidMoves(fstate,1)\n",
        "            self.Ps[s] = self.Ps[s]*valids\n",
        "            sPs = np.sum(self.Ps[s])\n",
        "            if sPs > 0:\n",
        "                self.Ps[s] = self.Ps[s]/sPs\n",
        "            else:\n",
        "                self.Ps[s] = self.Ps[s] + valids\n",
        "                self.Ps[s] = self.Ps[s]/np.sum(self.Ps[s])\n",
        "            \n",
        "            self.Vs[s] = valids\n",
        "            self.Ns[s] = 0\n",
        "            return -v\n",
        "        \n",
        "        valids = self.Vs[s]\n",
        "        best = -float('inf')\n",
        "        best_act = -1\n",
        "        for a in range(7):\n",
        "            if valids[a]:\n",
        "                if (s,a) in self.Qsa:\n",
        "                    PUCT = self.Qsa[(s,a)] + self.cput * self.Ps[s][a] * math.sqrt(self.Ns[s]) / (1+self.Nsa[(s,a)])\n",
        "                else:\n",
        "                    PUCT = self.cput * self.Ps[s][a] * math.sqrt(self.Ns[s])\n",
        "                \n",
        "                if PUCT > best:\n",
        "                    best = PUCT\n",
        "                    best_act = a\n",
        "        \n",
        "        a = best_act\n",
        "        next_state,next_player = self.game.Next(fstate,1,best_act)\n",
        "        next_fstate = self.game.fstategen(next_state,next_player)\n",
        "        \n",
        "        v = self.search(deepcopy(next_fstate))\n",
        "        \n",
        "        if (s, a) in self.Qsa:\n",
        "            self.Qsa[(s,a)]=(self.Nsa[(s,a)]*self.Qsa[(s,a)]+v)/(self.Nsa[(s,a)]+1)\n",
        "            self.Nsa[(s,a)]+=1\n",
        "\n",
        "        else:\n",
        "            self.Qsa[(s,a)]=v\n",
        "            self.Nsa[(s,a)]=1\n",
        "\n",
        "        self.Ns[s]+=1\n",
        "        return -v\n",
        "    \n",
        "    def getprobs(self,fstate):\n",
        "        nu=self.num\n",
        "        s = fstate.tobytes()\n",
        "        if s in self.Ns:\n",
        "            nu=nu-self.Ns[s]-1\n",
        "        for i in range(nu):\n",
        "            self.search(deepcopy(fstate))\n",
        "        \n",
        "        probs = [self.Nsa[(s, a)] if (s, a) in self.Nsa else 0 for a in range(7)]\n",
        "        #print(probs)\n",
        "        probSum = float(sum(probs))\n",
        "        if probSum==0:\n",
        "            return probs\n",
        "        else:\n",
        "            probs = [x/probSum for x in probs]\n",
        "        return probs\n",
        "    \n",
        "    def getAction(self,fstate):\n",
        "        probs=self.getprobs(fstate)\n",
        "        best=-float('inf')\n",
        "        best_act=-1\n",
        "        for a in range(len(probs)):\n",
        "            if probs[a]>best:\n",
        "                best=probs[a]\n",
        "                best_act=a\n",
        "        return best_act"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ed47DKYtcxZR"
      },
      "source": [
        "### selfplay class to Train the agent"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oNGPJxyBrYyn"
      },
      "source": [
        "import numpy as np\n",
        "import random\n",
        "\n",
        "class selfplay():\n",
        "    def __init__(self):\n",
        "        pass\n",
        "        \n",
        "    def play(self,toprint,mu):\n",
        "        examples = []\n",
        "        self.game.state=np.zeros((6,7))\n",
        "        self.game.player=1\n",
        "        self.mcts=Mcts(self.game,self.nn,self.cput,self.num)\n",
        "        while True:\n",
        "            #self.mcts=Mcts(self.game,self.nn,self.cput,self.num)\n",
        "            \n",
        "            if toprint:\n",
        "                print(\"\"\"\n",
        "                ##########################################\n",
        "                ##########################################\n",
        "                \"\"\")\n",
        "                self.game.PrintBoard(self.game.state)\n",
        "                print(\"\")\n",
        "            \n",
        "            fstate = self.game.fstategen(self.game.state,self.game.player)\n",
        "            pi = np.array(self.mcts.getprobs(deepcopy(fstate)))\n",
        "            pro=np.exp(pi*mu)*self.game.ValidMoves(fstate,1)\n",
        "            pro=pro/np.sum(pro)\n",
        "            if toprint:\n",
        "                print(pi)\n",
        "                print(pro)\n",
        "            best_act=np.random.choice(7,p=pro)\n",
        "            #best_act=np.random.choice(7,p=pi)\n",
        "            examples.append([deepcopy(self.game.state),self.game.player,deepcopy(pi)])\n",
        "\n",
        "            self.game.play(best_act)\n",
        "            r = self.game.end(self.game.state,self.game.player)\n",
        "            if r!=-2:\n",
        "                if toprint:\n",
        "                    self.game.PrintBoard(self.game.state)\n",
        "                    print(\"\"\"\n",
        "                    ---------------------------------------------------------------------\n",
        "                    ---------------------------------------------------------------------\n",
        "                    \"\"\")\n",
        "                return [[x[0]*x[1] for x in examples], [r*self.game.player*x[1] for x in examples], [x[2] for x in examples]]\n",
        "\n",
        "            #examples.append([deepcopy(self.game.state),self.game.player,deepcopy(pi)])\n",
        "    \n",
        "    def learn(self,game,nn,cput,num,no_of_games, toprint=False, mu=30):\n",
        "        self.game=game\n",
        "        self.nn=nn\n",
        "        self.cput=cput\n",
        "        self.num=num\n",
        "        self.no_of_games=no_of_games\n",
        "        self.mcts=Mcts(self.game,self.nn,self.cput,self.num)\n",
        "        X, Y1, Y2 = [],[],[]\n",
        "        for _ in range(self.no_of_games):\n",
        "            print(\"no_of_games\",_)\n",
        "            x,y1,y2=self.play(toprint,mu)\n",
        "            X+=x\n",
        "            Y1+=y1\n",
        "            Y2+=y2\n",
        "            # for i in range(len(X)):\n",
        "                # print(X[i])\n",
        "                # print(Y1[i])\n",
        "                # print(Y2[i])\n",
        "            \n",
        "        self.nn.train(X,Y1,Y2,10)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "szaRYXtPdAEY"
      },
      "source": [
        "# Training phase"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zjkwM9pKvZHn"
      },
      "source": [
        "game = Game()\n",
        "nn = Network()\n",
        "p = selfplay()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-pD_V2ydVurT",
        "outputId": "24845206-42ba-40f8-8d59-3c8c879d1fdf"
      },
      "source": [
        "p.learn(game,nn,1,20,10,False)\n",
        "p.learn(game,nn,1,20,10,False)\n",
        "#p.learn(game,nn,1,20,10,False)\n",
        "#p.learn(game,nn,1,20,10,False)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "no_of_games 0\n",
            "no_of_games 1\n",
            "no_of_games 2\n",
            "no_of_games 3\n",
            "no_of_games 4\n",
            "no_of_games 5\n",
            "no_of_games 6\n",
            "no_of_games 7\n",
            "no_of_games 8\n",
            "no_of_games 9\n",
            "Epoch 1/10\n",
            "3/3 [==============================] - 0s 135ms/step - loss: 3.2549 - valuehead_loss: 1.2834 - policyhead_loss: 1.9716\n",
            "Epoch 2/10\n",
            "3/3 [==============================] - 0s 8ms/step - loss: 3.0494 - valuehead_loss: 1.0952 - policyhead_loss: 1.9541\n",
            "Epoch 3/10\n",
            "3/3 [==============================] - 0s 8ms/step - loss: 2.9721 - valuehead_loss: 1.0094 - policyhead_loss: 1.9626\n",
            "Epoch 4/10\n",
            "3/3 [==============================] - 0s 8ms/step - loss: 2.9149 - valuehead_loss: 0.9495 - policyhead_loss: 1.9654\n",
            "Epoch 5/10\n",
            "3/3 [==============================] - 0s 8ms/step - loss: 2.8648 - valuehead_loss: 0.9124 - policyhead_loss: 1.9524\n",
            "Epoch 6/10\n",
            "3/3 [==============================] - 0s 8ms/step - loss: 2.8253 - valuehead_loss: 0.8772 - policyhead_loss: 1.9481\n",
            "Epoch 7/10\n",
            "3/3 [==============================] - 0s 8ms/step - loss: 2.7532 - valuehead_loss: 0.8073 - policyhead_loss: 1.9459\n",
            "Epoch 8/10\n",
            "3/3 [==============================] - 0s 8ms/step - loss: 2.6609 - valuehead_loss: 0.7136 - policyhead_loss: 1.9473\n",
            "Epoch 9/10\n",
            "3/3 [==============================] - 0s 9ms/step - loss: 2.5115 - valuehead_loss: 0.5698 - policyhead_loss: 1.9417\n",
            "Epoch 10/10\n",
            "3/3 [==============================] - 0s 8ms/step - loss: 2.5901 - valuehead_loss: 0.6484 - policyhead_loss: 1.9417\n",
            "no_of_games 0\n",
            "no_of_games 1\n",
            "no_of_games 2\n",
            "no_of_games 3\n",
            "no_of_games 4\n",
            "no_of_games 5\n",
            "no_of_games 6\n",
            "no_of_games 7\n",
            "no_of_games 8\n",
            "no_of_games 9\n",
            "Epoch 1/10\n",
            "3/3 [==============================] - 0s 127ms/step - loss: 3.2592 - valuehead_loss: 1.3120 - policyhead_loss: 1.9471\n",
            "Epoch 2/10\n",
            "3/3 [==============================] - 0s 7ms/step - loss: 2.9943 - valuehead_loss: 1.0507 - policyhead_loss: 1.9437\n",
            "Epoch 3/10\n",
            "3/3 [==============================] - 0s 7ms/step - loss: 2.8595 - valuehead_loss: 0.9141 - policyhead_loss: 1.9454\n",
            "Epoch 4/10\n",
            "3/3 [==============================] - 0s 8ms/step - loss: 2.8933 - valuehead_loss: 0.9512 - policyhead_loss: 1.9420\n",
            "Epoch 5/10\n",
            "3/3 [==============================] - 0s 8ms/step - loss: 2.8875 - valuehead_loss: 0.9471 - policyhead_loss: 1.9404\n",
            "Epoch 6/10\n",
            "3/3 [==============================] - 0s 8ms/step - loss: 2.7550 - valuehead_loss: 0.8060 - policyhead_loss: 1.9490\n",
            "Epoch 7/10\n",
            "3/3 [==============================] - 0s 7ms/step - loss: 2.5926 - valuehead_loss: 0.6383 - policyhead_loss: 1.9543\n",
            "Epoch 8/10\n",
            "3/3 [==============================] - 0s 7ms/step - loss: 2.6148 - valuehead_loss: 0.6671 - policyhead_loss: 1.9478\n",
            "Epoch 9/10\n",
            "3/3 [==============================] - 0s 8ms/step - loss: 2.5244 - valuehead_loss: 0.5769 - policyhead_loss: 1.9475\n",
            "Epoch 10/10\n",
            "3/3 [==============================] - 0s 8ms/step - loss: 2.4479 - valuehead_loss: 0.4991 - policyhead_loss: 1.9489\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hn7RfZOSt5Io"
      },
      "source": [
        "nn.new_champ()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1nuaD5eddEQQ"
      },
      "source": [
        "# The champion vs contendor to evaluate the agent's skills (Incomplete)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GJ2kznSQBCzr"
      },
      "source": [
        "class MctsChamp():\n",
        "    def __init__(self,game,nn,cput,num):\n",
        "        self.game = game\n",
        "        self.nn = nn\n",
        "        self.cput = cput\n",
        "        self.num = num\n",
        "        self.Qsa = {}\n",
        "        self.Nsa = {}\n",
        "        self.Ns = {}\n",
        "        self.Ps = {}\n",
        "        self.Es = {}\n",
        "        self.Vs = {}\n",
        "    \n",
        "    def search(self,fstate):\n",
        "        s = fstate.tobytes()\n",
        "        \n",
        "        if s not in self.Es:\n",
        "            self.Es[s] = self.game.end(fstate,1)\n",
        "        if self.Es[s]!=-2:\n",
        "            return -self.Es[s]\n",
        "        \n",
        "        if s not in self.Ps:\n",
        "            [v,tempP] = self.nn.champ_predict(fstate,1)\n",
        "            v = v[0][0]\n",
        "            self.Ps[s] = tempP[0]\n",
        "            valids = self.game.ValidMoves(fstate,1)\n",
        "            self.Ps[s] = self.Ps[s]*valids\n",
        "            sPs = np.sum(self.Ps[s])\n",
        "            if sPs > 0:\n",
        "                self.Ps[s] = self.Ps[s]/sPs\n",
        "            else:\n",
        "                self.Ps[s] = self.Ps[s] + valids\n",
        "                self.Ps[s] = self.Ps[s]/np.sum(self.Ps[s])\n",
        "            \n",
        "            self.Vs[s] = valids\n",
        "            self.Ns[s] = 0\n",
        "            return -v\n",
        "        \n",
        "        valids = self.Vs[s]\n",
        "        best = -float('inf')\n",
        "        best_act = -1\n",
        "        for a in range(7):\n",
        "            if valids[a]:\n",
        "                if (s,a) in self.Qsa:\n",
        "                    PUCT = self.Qsa[(s,a)] + self.cput * self.Ps[s][a] * math.sqrt(self.Ns[s]) / (1+self.Nsa[(s,a)])\n",
        "                else:\n",
        "                    PUCT = self.cput * self.Ps[s][a] * math.sqrt(self.Ns[s])\n",
        "                \n",
        "                if PUCT > best:\n",
        "                    best = PUCT\n",
        "                    best_act = a\n",
        "        \n",
        "        a = best_act\n",
        "        next_state,next_player = self.game.Next(fstate,1,best_act)\n",
        "        next_fstate = self.game.fstategen(next_state,next_player)\n",
        "        \n",
        "        v = self.search(deepcopy(next_fstate))\n",
        "        \n",
        "        if (s, a) in self.Qsa:\n",
        "            self.Qsa[(s,a)]=(self.Nsa[(s,a)]*self.Qsa[(s,a)]+v)/(self.Nsa[(s,a)]+1)\n",
        "            self.Nsa[(s,a)]+=1\n",
        "\n",
        "        else:\n",
        "            self.Qsa[(s,a)]=v\n",
        "            self.Nsa[(s,a)]=1\n",
        "\n",
        "        self.Ns[s]+=1\n",
        "        return -v\n",
        "    \n",
        "    def getprobs(self,fstate):\n",
        "        nu=self.num\n",
        "        s = fstate.tobytes()\n",
        "        if s in self.Ns:\n",
        "            nu=nu-self.Ns[s]-1\n",
        "        for i in range(nu):\n",
        "            self.search(deepcopy(fstate))\n",
        "        \n",
        "        probs = [self.Nsa[(s, a)] if (s, a) in self.Nsa else 0 for a in range(7)]\n",
        "        #print(probs)\n",
        "        probSum = float(sum(probs))\n",
        "        if probSum==0:\n",
        "            return probs\n",
        "        else:\n",
        "            probs = [x/probSum for x in probs]\n",
        "        return probs\n",
        "    \n",
        "    def getAction(self,fstate):\n",
        "        probs=self.getprobs(fstate)\n",
        "        best=-float('inf')\n",
        "        best_act=-1\n",
        "        for a in range(len(probs)):\n",
        "            if probs[a]>best:\n",
        "                best=probs[a]\n",
        "                best_act=a\n",
        "        return best_act"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mGMQ9IhnOoFB"
      },
      "source": [
        "def championship(game,para=[]):\n",
        "    result=[]\n",
        "    turn=0\n",
        "    for turn in range(2):\n",
        "        player1=Mcts(*para)\n",
        "        player2=MctsChamp(*para)\n",
        "        game.state=np.zeros((6,7))\n",
        "        game.player=1\n",
        "        while True:\n",
        "            #if toprint:\n",
        "            print(\"\"\"\n",
        "            ##########################################\n",
        "            \"\"\")\n",
        "            game.PrintBoard(game.state)\n",
        "            # print(\"\"\"\n",
        "            # ##########################################\n",
        "            # ##########################################\n",
        "            # \"\"\")\n",
        "            fstate = game.fstategen(game.state,game.player)\n",
        "            if turn%2==0:\n",
        "                print(\"contendor\")\n",
        "                best_act=player1.getAction(deepcopy(fstate))\n",
        "            else:\n",
        "                print(\"champ\")\n",
        "                best_act=player2.getAction(deepcopy(fstate))\n",
        "            turn+=1\n",
        "            game.play(best_act)\n",
        "            r = game.end(game.state,1)\n",
        "            if r!=-2:\n",
        "                game.PrintBoard(game.state)\n",
        "                print(\"\"\"\n",
        "                ---------------------------------------------------------------------\n",
        "                ---------------------------------------------------------------------\n",
        "                \"\"\")\n",
        "                print(r)\n",
        "                result.append((r*((-1)**turn)))\n",
        "                break\n",
        "\n",
        "        player1,player2=player2,player1\n",
        "    print(result)\n",
        "    return result"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NbeRoLi544tq"
      },
      "source": [
        "class playWith():\n",
        "    def __init__(self):\n",
        "        pass\n",
        "        \n",
        "    def play(self,game,nn,cput,num,pos):\n",
        "        self.game=deepcopy(game)\n",
        "        self.nn=nn\n",
        "        self.cput=cput\n",
        "        self.num=num\n",
        "        self.game.state=np.zeros((6,7))\n",
        "        self.game.player=1\n",
        "        self.mcts=Mcts(self.game,self.nn,self.cput,self.num)\n",
        "        examples = []\n",
        "        self.game.PrintBoard(self.game.state)\n",
        "        print(\"\"\"\n",
        "        ##########################################\n",
        "        ##########################################\n",
        "        \"\"\")\n",
        "            \n",
        "        while True:            \n",
        "\n",
        "            if self.game.player==(-1)*pos:\n",
        "                print(\"AI's Turn\")\n",
        "                print(\"Selecting Action........\")\n",
        "                print(\" 0 1 2 3 4 5 6 \")            \n",
        "                fstate = self.game.fstategen(self.game.state,self.game.player)\n",
        "                pi = self.mcts.getprobs(deepcopy(fstate))\n",
        "                best_act = self.mcts.getAction(deepcopy(fstate))\n",
        "                examples.append([deepcopy(self.game.state),self.game.player,deepcopy(pi)])\n",
        "                self.game.play(best_act)\n",
        "                self.game.PrintBoard(self.game.state)\n",
        "                print(\"\"\"\n",
        "                ##########################################\n",
        "                ##########################################\n",
        "                \"\"\")\n",
        "            \n",
        "            else:\n",
        "                print(\"Your Turn\")\n",
        "                best_act = int(input(\"Your Action:\"))\n",
        "                print(\" 0 1 2 3 4 5 6 \")\n",
        "                self.game.play(best_act)\n",
        "                self.game.PrintBoard(self.game.state)\n",
        "                print(\"\"\"\n",
        "                ##########################################\n",
        "                ##########################################\n",
        "                \"\"\")\n",
        "                \n",
        "            r = self.game.end(self.game.state,self.game.player)\n",
        "            if r!=-2:\n",
        "                print(\"Game Ended!!!\")\n",
        "                if self.game.player==pos:\n",
        "                    if r==1:\n",
        "                        print(\"You Won!!!\")\n",
        "                    if r==-1:\n",
        "                        print(\"You Lost!!!\")\n",
        "                    if r==0:\n",
        "                        print(\"It is a draw!!!\")\n",
        "                else:\n",
        "                    if r==1:\n",
        "                        print(\"You Lost!!!\")\n",
        "                    if r==-1:\n",
        "                        print(\"You Won!!!\")\n",
        "                    if r==0:\n",
        "                        print(\"It is a draw!!!\")\n",
        "                return [[x[0]*x[1] for x in examples], [r*self.game.player*x[1] for x in examples], [x[2] for x in examples]]\n",
        "                #return 1, 2, 3"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cR4SAQ145Nbu"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}