scripts/3c._Classical_DQL_sim_quant_finetuning.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gJenL7duYhPG",
        "outputId": "2c586ea3-088f-4fbc-b81d-06e5a9c32fb5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mon May 23 17:32:06 2022       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 512.15       Driver Version: 512.15       CUDA Version: 11.6     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name            TCC/WDDM | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  NVIDIA GeForce ... WDDM  | 00000000:02:00.0  On |                  N/A |\n",
            "|  0%   39C    P8     5W / 190W |    456MiB /  6144MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|    0   N/A  N/A       556    C+G   ...2txyewy\\TextInputHost.exe    N/A      |\n",
            "|    0   N/A  N/A       780    C+G                                   N/A      |\n",
            "|    0   N/A  N/A      4632    C+G   C:\\Windows\\explorer.exe         N/A      |\n",
            "|    0   N/A  N/A      4988    C+G   ...bbwe\\Microsoft.Photos.exe    N/A      |\n",
            "|    0   N/A  N/A      5228    C+G   ...perience\\NVIDIA Share.exe    N/A      |\n",
            "|    0   N/A  N/A      7344    C+G   ...8wekyb3d8bbwe\\GameBar.exe    N/A      |\n",
            "|    0   N/A  N/A      7404    C+G   ...artMenuExperienceHost.exe    N/A      |\n",
            "|    0   N/A  N/A      8512    C+G   ...y\\ShellExperienceHost.exe    N/A      |\n",
            "|    0   N/A  N/A      8940    C+G   ...cw5n1h2txyewy\\LockApp.exe    N/A      |\n",
            "|    0   N/A  N/A     10900    C+G   ...n64\\EpicGamesLauncher.exe    N/A      |\n",
            "|    0   N/A  N/A     11148    C+G   ...s\\Win64\\EpicWebHelper.exe    N/A      |\n",
            "|    0   N/A  N/A     12904    C+G   ...ician\\SamsungMagician.exe    N/A      |\n",
            "|    0   N/A  N/A     13292    C+G   ...icrosoft VS Code\\Code.exe    N/A      |\n",
            "|    0   N/A  N/A     13756    C+G   ...me\\Application\\chrome.exe    N/A      |\n",
            "|    0   N/A  N/A     15696    C+G   ...zpdnekdrzrea0\\Spotify.exe    N/A      |\n",
            "|    0   N/A  N/A     17020    C+G   ...lPanel\\SystemSettings.exe    N/A      |\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ],
      "source": [
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q5kdrvKBMAqb",
        "outputId": "24016870-121d-4fde-f409-d45edf7d35fc"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: pygame in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (2.1.2)\n"
          ]
        }
      ],
      "source": [
        "!pip install pygame"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 537
        },
        "id": "loapReo7Li9N",
        "outputId": "2121c99a-fa55-4c74-d2e8-111d65957399"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: gym in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (0.23.1)\n",
            "Requirement already satisfied: cloudpickle>=1.2.0 in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (from gym) (2.0.0)\n",
            "Requirement already satisfied: importlib-metadata>=4.10.0 in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (from gym) (4.11.3)\n",
            "Requirement already satisfied: gym-notices>=0.0.4 in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (from gym) (0.0.6)\n",
            "Requirement already satisfied: numpy>=1.18.0 in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (from gym) (1.21.5)\n",
            "Requirement already satisfied: zipp>=0.5 in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (from importlib-metadata>=4.10.0->gym) (3.8.0)\n"
          ]
        }
      ],
      "source": [
        "!pip install --upgrade gym"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PFYNrFRxwxmC",
        "outputId": "3cfc5789-ca4d-437d-e318-40260159c041"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: ray==1.9 in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (1.9.0)\n",
            "Requirement already satisfied: attrs in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (from ray==1.9) (21.4.0)\n",
            "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (from ray==1.9) (1.0.3)\n",
            "Requirement already satisfied: protobuf>=3.15.3 in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (from ray==1.9) (3.20.1)\n",
            "Requirement already satisfied: numpy>=1.19.3 in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (from ray==1.9) (1.21.5)\n",
            "Requirement already satisfied: redis>=3.5.0 in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (from ray==1.9) (4.3.1)\n",
            "Requirement already satisfied: click>=7.0 in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (from ray==1.9) (8.1.3)\n",
            "Requirement already satisfied: grpcio>=1.28.1 in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (from ray==1.9) (1.43.0)\n",
            "Requirement already satisfied: filelock in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (from ray==1.9) (3.7.0)\n",
            "Requirement already satisfied: jsonschema in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (from ray==1.9) (4.4.0)\n",
            "Requirement already satisfied: pyyaml in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (from ray==1.9) (6.0)\n",
            "Requirement already satisfied: colorama in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (from click>=7.0->ray==1.9) (0.4.4)\n",
            "Requirement already satisfied: six>=1.5.2 in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (from grpcio>=1.28.1->ray==1.9) (1.16.0)\n",
            "Requirement already satisfied: async-timeout>=4.0.2 in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (from redis>=3.5.0->ray==1.9) (4.0.2)\n",
            "Requirement already satisfied: deprecated>=1.2.3 in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (from redis>=3.5.0->ray==1.9) (1.2.13)\n",
            "Requirement already satisfied: packaging>=20.4 in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (from redis>=3.5.0->ray==1.9) (21.3)\n",
            "Requirement already satisfied: wrapt<2,>=1.10 in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (from deprecated>=1.2.3->redis>=3.5.0->ray==1.9) (1.14.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (from packaging>=20.4->redis>=3.5.0->ray==1.9) (3.0.7)\n",
            "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in c:\\users\\majus\\.conda\\envs\\qtorchgpuauto\\lib\\site-packages (from jsonschema->ray==1.9) (0.18.1)\n"
          ]
        }
      ],
      "source": [
        "!pip install ray==1.9"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Cv3IzSOVwx1T"
      },
      "source": [
        "# Imports:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XCg9x-yQuCek",
        "outputId": "22f01b42-a867-487b-9fb2-9a4a571de57d"
      },
      "outputs": [],
      "source": [
        "import gym\n",
        "import numpy as np \n",
        "import os\n",
        "from ray import tune\n",
        "from ray.tune import JupyterNotebookReporter\n",
        "from ray.tune.schedulers import ASHAScheduler\n",
        "from functools import partial\n",
        "\n",
        "import torch\n",
        "from torch import nn\n",
        "from torch import optim\n",
        "from torch.nn import functional as F\n",
        "from torch import linalg as LA\n",
        "\n",
        "from src.visualizations import *\n",
        "from src.entropies import entanglement_entropy, classical_entropy\n",
        "from src.utils import one_hot, uniform_linear_layer"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1F2GmhVRxIMq"
      },
      "source": [
        "## Environment setup:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "W9yeNkcNxLyk"
      },
      "outputs": [],
      "source": [
        "os.environ[\"KMP_DUPLICATE_LIB_OK\"]=\"TRUE\"\n",
        "\n",
        "activation_function = 'sigmoid'\n",
        "global_seed = None\n",
        "\n",
        "max_steps = 60\n",
        "non_random_chance = 0.99\n",
        "window = 100\n",
        "target_win_ratio = 0.70\n",
        "min_steps_num = 6"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Flh5RnCdSxiw"
      },
      "source": [
        "## Helpers:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "XoTF7lHISxH8"
      },
      "outputs": [],
      "source": [
        "class Agent(nn.Module):\n",
        "    def __init__(self, observation_space_size, n_hidden_layers, activation_function):\n",
        "        super(Agent, self).__init__()\n",
        "        self.observation_space_size = observation_space_size\n",
        "        self.hidden_size = 2*self.observation_space_size\n",
        "\n",
        "        self.l1 = nn.Linear(in_features=2*self.observation_space_size, out_features=self.hidden_size)\n",
        "        self.hidden_layers = [\n",
        "            nn.Linear(in_features=self.hidden_size, out_features=self.hidden_size) \\\n",
        "                for i in range(n_hidden_layers)\n",
        "        ]\n",
        "        self.l2 = nn.Linear(in_features=self.hidden_size, out_features=32) \n",
        "        self.activation = None\n",
        "        if activation_function=='lrelu':\n",
        "            self.activation = torch.leaky_relu\n",
        "        if activation_function=='sigmoid':\n",
        "            self.activation = torch.sigmoid\n",
        "        if activation_function=='tanh':\n",
        "            self.activation = torch.tanh\n",
        "\n",
        "        uniform_linear_layer(self.l1)\n",
        "        for l in self.hidden_layers:\n",
        "            uniform_linear_layer(l)\n",
        "\n",
        "        uniform_linear_layer(self.l2)\n",
        "    \n",
        "    def forward(self, state):\n",
        "        obs_emb = one_hot([int(2*state)], 2*self.observation_space_size)\n",
        "        # first layer:\n",
        "        out1 = self.activation(self.l1(obs_emb))\n",
        "        \n",
        "        # hidden layers:\n",
        "        for l in self.hidden_layers:\n",
        "            out1 = self.activation(l(out1))\n",
        "        \n",
        "        # output layers:\n",
        "        out2 = self.activation(self.l2(out1))\n",
        "\n",
        "        return out2.view((-1)) \n",
        "\n",
        "\n",
        "class Trainer:\n",
        "    def __init__(self, n_hidden_layers, lake, learning_rate, non_random_chance, random_scaling, gamma, activation_function):\n",
        "        self.holes_indexes = np.array([5,7,11,12])\n",
        "\n",
        "        self.lake = lake\n",
        "        self.agent = Agent(self.lake.observation_space.n, n_hidden_layers, activation_function)\n",
        "        self.optimizer = optim.Adam(params=self.agent.parameters(), lr=learning_rate)\n",
        "        \n",
        "        self.epsilon = non_random_chance\n",
        "        self.epsilon_growth_rate = random_scaling\n",
        "        self.gamma = gamma\n",
        "        \n",
        "        self.epsilon_list = []\n",
        "        self.success = []\n",
        "        self.jList = []\n",
        "        self.reward_list = []\n",
        "\n",
        "        self.compute_entropy = False\n",
        "        self.entropies = []\n",
        "        self.cl_entropies = []\n",
        "        self.entropies_episodes = [0]\n",
        "        \n",
        "        self.print = False\n",
        "\n",
        "    \n",
        "    def train(self, epoch, max_steps, window, target_win_ratio):\n",
        "        # entropies_episodes = [0] * (epoch+1)\n",
        "        #pbar = tqdm(range(epoch))\n",
        "        for i in range(epoch):\n",
        "            #pbar.set_description(f'Success rate: {sum(self.success[-window:])/window:.2%} | Random chance: {self.epsilon:.2%}')\n",
        "            \n",
        "            s = self.lake.reset() #stan na jeziorze 0-16, dla resetu 0\n",
        "            j = 0\n",
        "            self.entropies_episodes.append(0)\n",
        "            while j < max_steps:\n",
        "                j += 1\n",
        "                # perform chosen action\n",
        "                a = self.choose_action(s)\n",
        "                s1, r, d, _ = self.lake.step(int(a))\n",
        "                if d == True and r == 0: r = -1\n",
        "                elif d== True: r == 1\n",
        "                elif r==0: r = -0.01\n",
        "\n",
        "                # if self.print==False:\n",
        "                #     print(self.agent(s)[a])\n",
        "                #     self.print=True\n",
        "\n",
        "                # calculate target and loss\n",
        "                target_q = r + self.gamma * torch.max(self.calc_probabilities(s1).detach()) \n",
        "\n",
        "                loss = F.smooth_l1_loss(self.calc_probability(s, a), target_q) \n",
        "                # update model to optimize Q\n",
        "                self.optimizer.zero_grad()\n",
        "                loss.backward()\n",
        "                self.optimizer.step()\n",
        "                \n",
        "                # update state\n",
        "                s = s1\n",
        "                if(self.compute_entropy):\n",
        "                    with torch.inference_mode():\n",
        "                        self.entropies.append(entanglement_entropy(self.calc_statevector(s))) \n",
        "                        self.cl_entropies.append(classical_entropy(self.calc_statevector(s))) \n",
        "                        self.entropies_episodes[i] += 1\n",
        "                \n",
        "                if d == True: break\n",
        "            \n",
        "            # append results onto report lists\n",
        "            if d == True and r > 0:\n",
        "                self.success.append(1)\n",
        "            else:\n",
        "                self.success.append(0)\n",
        "\n",
        "            self.reward_list.append(r)\n",
        "            self.jList.append(j)\n",
        "\n",
        "            if self.epsilon < 1.:\n",
        "                self.epsilon *= self.epsilon_growth_rate\n",
        "            self.epsilon_list.append(self.epsilon)\n",
        "\n",
        "            tune.report(\n",
        "                if_trained = sum(self.success[-window:])/window>target_win_ratio, \n",
        "                win_ratio = sum(self.success[-window:])/window,\n",
        "                episode_reward_mean = sum(self.reward_list[-window:])/window,\n",
        "                steps = len(self.success)\n",
        "                )\n",
        "\n",
        "            if i>100:\n",
        "                if sum(self.success[-window:])/window>target_win_ratio:\n",
        "                    #print(\"Network trained before epoch limit on {i} epoch\".format(i=i))\n",
        "                    break\n",
        "\n",
        "        #print(\"last 100 epoches success rate: \" + str(sum(self.success[-100:])/100) + \"%\")\n",
        "\n",
        "    def choose_action(self, s):\n",
        "        self.calc_probabilities(s)\n",
        "        if np.random.rand(1) > self.epsilon : \n",
        "            action = torch.argmax(self.calc_probabilities(s)) #wybor najwiekszej wartosci z tablicy\n",
        "        else:\n",
        "            action = torch.tensor(np.random.randint(0, 4))\n",
        "        return action\n",
        "    \n",
        "    def calc_statevector(self, s):\n",
        "        return torch.complex(self.agent(s)[0::2], self.agent(s)[1::2])\n",
        "\n",
        "    def calc_probability(self, s, a): #liczenie prawdopodobieństwa obsadzenia kubitu (0-3) z danego stanu planszy (0-15)\n",
        "        statevector = torch.complex(self.agent(s)[0::2], self.agent(s)[1::2])\n",
        "        probabilities = (statevector.abs()**2)\n",
        "        probabilities = probabilities/probabilities.sum() #normowanie\n",
        "        prob_indexes = [\n",
        "            [0,1,2,3,4,5,6,7],\n",
        "            [0,1,2,3,8,9,10,11],\n",
        "            [0,1,4,5,8,9,12,13],\n",
        "            [0,2,4,6,8,10,12,14]\n",
        "        ]\n",
        "        return probabilities[prob_indexes[a]].sum()\n",
        "\n",
        "    def calc_probabilities(self, s): #liczenie prawdopodobieństw każdego z kubitów z danego stanu planszy (0-15) do tensora o kształcie (4)\n",
        "        raw_wavefunction = torch.complex(self.agent(s)[0::2], self.agent(s)[1::2])\n",
        "        probabilities = (raw_wavefunction.abs()**2)\n",
        "        probabilities = probabilities/probabilities.sum() #normowanie\n",
        "        probs_of_qubits = torch.tensor([\n",
        "            probabilities[[0,1,2,3,4,5,6,7]].sum(),\n",
        "            probabilities[[0,1,2,3,8,9,10,11]].sum(),\n",
        "            probabilities[[0,1,4,5,8,9,12,13]].sum(),\n",
        "            probabilities[[0,2,4,6,8,10,12,14]].sum()\n",
        "            ])\n",
        "        return probs_of_qubits\n",
        "\n",
        "        \n",
        "    def Q(self):\n",
        "        Q = []\n",
        "        for x in range(self.lake.observation_space.n):\n",
        "            Qstate = self.agent(x).detach()\n",
        "            Qstate /= LA.norm(Qstate)\n",
        "            Q.append(Qstate)   \n",
        "        Q_out = torch.Tensor(self.lake.observation_space.n, self.lake.action_space.n)\n",
        "        torch.cat(Q, out=Q_out)\n",
        "        return Q_out\n",
        "    \n",
        "    def Qstate(self, state):\n",
        "        Qstate = self.agent(state).detach()\n",
        "        Qstate /= LA.norm(Qstate)\n",
        "        return Qstate\n",
        "    \n",
        "    def Qstrategy(self):\n",
        "        return [torch.argmax(self.calc_probabilities(state)).item() for state in range(self.lake.observation_space.n)]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DBusRPO8xBGj"
      },
      "source": [
        "# Ray wrapper:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "kZk8N7H8dP_N"
      },
      "outputs": [],
      "source": [
        "def perform_experiment(config):\n",
        "    # torch.manual_seed(global_seed)\n",
        "    # np.random.seed(global_seed)\n",
        "\n",
        "    results_path = os.path.join('.', 'visuals')\n",
        "    \n",
        "    if not os.path.exists(results_path):\n",
        "        os.mkdir(results_path)\n",
        "\n",
        "    lake = gym.make('FrozenLake-v1', is_slippery=False)\n",
        "    lake.reset()\n",
        "\n",
        "    fl = Trainer(config[\"n_hidden_layers\"], lake, config[\"lr\"], non_random_chance, config[\"random_scaling\"], config[\"gamma\"], activation_function)\n",
        "\n",
        "    fl.train(epochs, max_steps, window, target_win_ratio)\n",
        "\n",
        "    plot_success_steps_history(fl.jList, fl.success)\n",
        "\n",
        "    strategy = np.array(fl.Qstrategy()).reshape((4,4))\n",
        "    strategy_save_path = os.path.join(results_path, \"trained_strategy.jpg\")\n",
        "    plot_strategy(strategy, fl.holes_indexes, strategy_save_path)\n",
        "\n",
        "    moving_average_history_save_path = os.path.join(results_path, \"training_history_moving_average.jpg\")\n",
        "    plot_rolling_window_history(fl.jList, fl.reward_list, fl.success, np.array(fl.epsilon_list), target_win_ratio, min_steps_num, moving_average_history_save_path, window=window)\n",
        "    history_save_path = os.path.join(results_path, \"training_history.jpg\")\n",
        "    plot_history(fl.jList, fl.reward_list, fl.success, np.array(fl.epsilon_list), target_win_ratio, min_steps_num, history_save_path)\n",
        "\n",
        "    with open(os.path.join(results_path, \"hyperparameters.txt\"), \"w+\") as f:\n",
        "        f.write(f'gamma;{config[\"gamma\"]}\\n')\n",
        "        f.write(f'epochs;{epochs}\\n')\n",
        "        f.write(f'max_steps;{max_steps}\\n')\n",
        "        f.write(f'learning_rate;{config[\"lr\"]}\\n')\n",
        "        f.write(f'non_random_chance;{non_random_chance}\\n')\n",
        "        f.write(f'random_scaling;{config[\"random_scaling\"]}\\n')\n",
        "        f.write(f'window;{window}\\n')\n",
        "        f.write(f'target_win_ratio;{target_win_ratio}\\n')\n",
        "        f.write(f'min_steps_num;{min_steps_num}\\n')\n",
        "        f.write(f'n_hidden_layers;{config[\"n_hidden_layers\"]}\\n')\n",
        "        f.write(f'activation_function;{activation_function}\\n')\n",
        "        f.write(f'global_seed;{global_seed}\\n')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HUldEs7YxRWc"
      },
      "source": [
        "# Experiments:"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e8-aTKbj9jy8"
      },
      "source": [
        "## Setup:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "08I3iGv_xRrP"
      },
      "outputs": [],
      "source": [
        "epochs = 30000\n",
        "\n",
        "config = {\n",
        "    \"n_hidden_layers\": tune.choice([1,2]),\n",
        "    \"lr\": tune.loguniform(1e-5, 4e-1),\n",
        "    \"random_scaling\": tune.loguniform(0.99999, 0.9995),\n",
        "    \"gamma\": tune.loguniform(2., 0.1)\n",
        "\n",
        "}\n",
        "scheduler = ASHAScheduler(\n",
        "    time_attr='steps',\n",
        "    metric='win_ratio',\n",
        "    mode='max',\n",
        "    max_t=epochs+1,\n",
        "    grace_period=15000,\n",
        "    reduction_factor=2\n",
        "    )\n",
        "\n",
        "\n",
        "reporter = JupyterNotebookReporter(\n",
        "    overwrite=True,\n",
        "    metric_columns=[\"if_trained\", \"win_ratio\", \"episode_reward_mean\", 'steps'],\n",
        "    parameter_columns=['steps'])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JMR8J0JF9mnk"
      },
      "source": [
        "## Run:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LEDFsFiB_fdc",
        "outputId": "eec87b60-549b-41ed-8347-3ab7043c2d9e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Resources used: CPU 1.0, GPU 0.16666666666666666 for each of 6 parallel runs.\n"
          ]
        }
      ],
      "source": [
        "import warnings\n",
        "\n",
        "\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "\n",
        "parallel_runs = 6\n",
        "#resources_usage = 0.8\n",
        "print(f'Resources used: CPU {6/parallel_runs}, GPU {1/parallel_runs} for each of {parallel_runs} parallel runs.')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "== Status ==<br>Current time: 2022-05-26 04:24:03 (running for 05:57:59.34)<br>Memory usage on this node: 8.7/14.9 GiB<br>Using AsyncHyperBand: num_stopped=2\n",
              "Bracket: Iter 30000.000: 0.0 | Iter 15000.000: 0.0<br>Resources requested: 0/6 CPUs, 0/1 GPUs, 0.0/5.83 GiB heap, 0.0/2.91 GiB objects<br>Result logdir: c:\\Users\\majus\\auto_ml\\full<br>Number of trials: 150/150 (150 TERMINATED)<br><table>\n",
              "<thead>\n",
              "<tr><th>Trial name         </th><th>status    </th><th>loc            </th><th>steps  </th><th>if_trained  </th><th style=\"text-align: right;\">  win_ratio</th><th style=\"text-align: right;\">  episode_reward_mean</th><th style=\"text-align: right;\">  steps</th></tr>\n",
              "</thead>\n",
              "<tbody>\n",
              "<tr><td>DEFAULT_08e8c1d1   </td><td>TERMINATED</td><td>127.0.0.1:11456</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08e84cf0   </td><td>TERMINATED</td><td>127.0.0.1:17124</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08e7b11f   </td><td>TERMINATED</td><td>127.0.0.1:12152</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.0298</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08e67972   </td><td>TERMINATED</td><td>127.0.0.1:17064</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08e62b89   </td><td>TERMINATED</td><td>127.0.0.1:1756 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08e6c75c   </td><td>TERMINATED</td><td>127.0.0.1:3588 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08e5dd9c   </td><td>TERMINATED</td><td>127.0.0.1:14876</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08e76334   </td><td>TERMINATED</td><td>127.0.0.1:2120 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.9604</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08e51ad2   </td><td>TERMINATED</td><td>127.0.0.1:5332 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08e4a5f2   </td><td>TERMINATED</td><td>127.0.0.1:15236</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_e64acd90   </td><td>TERMINATED</td><td>127.0.0.1:7760 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08e58fb6   </td><td>TERMINATED</td><td>127.0.0.1:1560 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.0397</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_e64b1b77   </td><td>TERMINATED</td><td>127.0.0.1:8684 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08e43112   </td><td>TERMINATED</td><td>127.0.0.1:8864 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08e20fa8   </td><td>TERMINATED</td><td>127.0.0.1:11140</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08e2f96c   </td><td>TERMINATED</td><td>127.0.0.1:10756</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08e3e327   </td><td>TERMINATED</td><td>127.0.0.1:15988</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.0991</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_e64b6962   </td><td>TERMINATED</td><td>127.0.0.1:2892 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_e64bb74c   </td><td>TERMINATED</td><td>127.0.0.1:12608</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_e64c0537   </td><td>TERMINATED</td><td>127.0.0.1:12084</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.0199</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08e2ab7d   </td><td>TERMINATED</td><td>127.0.0.1:12984</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.5446</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08e173d3   </td><td>TERMINATED</td><td>127.0.0.1:12356</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.307 </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_e64c5321   </td><td>TERMINATED</td><td>127.0.0.1:11124</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_e64ca10d   </td><td>TERMINATED</td><td>127.0.0.1:16928</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.01</td><td style=\"text-align: right;\">              -0.2672</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_e64ceefa   </td><td>TERMINATED</td><td>127.0.0.1:1504 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_e64d3ce3   </td><td>TERMINATED</td><td>127.0.0.1:15440</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_e64d8acd   </td><td>TERMINATED</td><td>127.0.0.1:9420 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_e64dd8b7   </td><td>TERMINATED</td><td>127.0.0.1:9416 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.0199</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08de41b0   </td><td>TERMINATED</td><td>127.0.0.1:12024</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08e0d801   </td><td>TERMINATED</td><td>127.0.0.1:11148</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.04</td><td style=\"text-align: right;\">              -0.92  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08e0631e   </td><td>TERMINATED</td><td>127.0.0.1:9196 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -1     </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08ddf3cb   </td><td>TERMINATED</td><td>127.0.0.1:9880 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08dedd88   </td><td>TERMINATED</td><td>127.0.0.1:15412</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -1     </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08dd7ee9   </td><td>TERMINATED</td><td>127.0.0.1:3340 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08dd30fe   </td><td>TERMINATED</td><td>127.0.0.1:7032 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08df2b72   </td><td>TERMINATED</td><td>127.0.0.1:16628</td><td>       </td><td>True        </td><td style=\"text-align: right;\">       0.71</td><td style=\"text-align: right;\">               0.42  </td><td style=\"text-align: right;\">   5081</td></tr>\n",
              "<tr><td>DEFAULT_08dc952f   </td><td>TERMINATED</td><td>127.0.0.1:2980 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.0694</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08dbf953   </td><td>TERMINATED</td><td>127.0.0.1:11752</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.0496</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08dce314   </td><td>TERMINATED</td><td>127.0.0.1:10544</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.604 </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08dbab68   </td><td>TERMINATED</td><td>127.0.0.1:6212 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -1     </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_08dc4742   </td><td>TERMINATED</td><td>127.0.0.1:16364</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -1     </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00148</td><td>TERMINATED</td><td>127.0.0.1:15996</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.0397</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00149</td><td>TERMINATED</td><td>127.0.0.1:15536</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.0595</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00142</td><td>TERMINATED</td><td>127.0.0.1:7480 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00146</td><td>TERMINATED</td><td>127.0.0.1:2432 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.8713</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00138</td><td>TERMINATED</td><td>127.0.0.1:12712</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00147</td><td>TERMINATED</td><td>127.0.0.1:14316</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.02</td><td style=\"text-align: right;\">              -0.96  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00143</td><td>TERMINATED</td><td>127.0.0.1:8652 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.0496</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00137</td><td>TERMINATED</td><td>127.0.0.1:4792 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.0298</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00131</td><td>TERMINATED</td><td>127.0.0.1:9656 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00123</td><td>TERMINATED</td><td>127.0.0.1:11356</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00136</td><td>TERMINATED</td><td>127.0.0.1:9964 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.03</td><td style=\"text-align: right;\">              -0.94  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00125</td><td>TERMINATED</td><td>127.0.0.1:1076 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00139</td><td>TERMINATED</td><td>127.0.0.1:6444 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -1     </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00127</td><td>TERMINATED</td><td>127.0.0.1:14608</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00121</td><td>TERMINATED</td><td>127.0.0.1:15340</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00124</td><td>TERMINATED</td><td>127.0.0.1:2556 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00130</td><td>TERMINATED</td><td>127.0.0.1:4228 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.2476</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00116</td><td>TERMINATED</td><td>127.0.0.1:132  </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00128</td><td>TERMINATED</td><td>127.0.0.1:16196</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.0694</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00122</td><td>TERMINATED</td><td>127.0.0.1:10736</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.307 </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00120</td><td>TERMINATED</td><td>127.0.0.1:1000 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.208 </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00129</td><td>TERMINATED</td><td>127.0.0.1:476  </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.3961</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00110</td><td>TERMINATED</td><td>127.0.0.1:1684 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00105</td><td>TERMINATED</td><td>127.0.0.1:11400</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00114</td><td>TERMINATED</td><td>127.0.0.1:13672</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -1     </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00112</td><td>TERMINATED</td><td>127.0.0.1:2804 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.41</td><td style=\"text-align: right;\">              -0.18  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00109</td><td>TERMINATED</td><td>127.0.0.1:6268 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -1     </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00111</td><td>TERMINATED</td><td>127.0.0.1:5068 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.15</td><td style=\"text-align: right;\">              -0.7   </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00098</td><td>TERMINATED</td><td>127.0.0.1:12600</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.16</td><td style=\"text-align: right;\">              -0.68  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00097</td><td>TERMINATED</td><td>127.0.0.1:2408 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -1     </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00107</td><td>TERMINATED</td><td>127.0.0.1:14512</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -1     </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00108</td><td>TERMINATED</td><td>127.0.0.1:15120</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -1     </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00103</td><td>TERMINATED</td><td>127.0.0.1:11692</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.03</td><td style=\"text-align: right;\">              -0.94  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00100</td><td>TERMINATED</td><td>127.0.0.1:16572</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.28</td><td style=\"text-align: right;\">              -0.44  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00106</td><td>TERMINATED</td><td>127.0.0.1:11928</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.02</td><td style=\"text-align: right;\">              -0.96  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00094</td><td>TERMINATED</td><td>127.0.0.1:2876 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.604 </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00090</td><td>TERMINATED</td><td>127.0.0.1:11568</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -1     </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00067</td><td>TERMINATED</td><td>127.0.0.1:7352 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00084</td><td>TERMINATED</td><td>127.0.0.1:13384</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -1     </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00066</td><td>TERMINATED</td><td>127.0.0.1:3604 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00064</td><td>TERMINATED</td><td>127.0.0.1:14172</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00085</td><td>TERMINATED</td><td>127.0.0.1:16748</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -1     </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00083</td><td>TERMINATED</td><td>127.0.0.1:17288</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.4 </td><td style=\"text-align: right;\">              -0.2   </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00076</td><td>TERMINATED</td><td>127.0.0.1:1260 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.08</td><td style=\"text-align: right;\">              -0.84  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00082</td><td>TERMINATED</td><td>127.0.0.1:8304 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.02</td><td style=\"text-align: right;\">              -0.96  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00070</td><td>TERMINATED</td><td>127.0.0.1:10808</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.7525</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00058</td><td>TERMINATED</td><td>127.0.0.1:9620 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00068</td><td>TERMINATED</td><td>127.0.0.1:15076</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.9208</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00055</td><td>TERMINATED</td><td>127.0.0.1:17256</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00071</td><td>TERMINATED</td><td>127.0.0.1:14604</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -1     </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00049</td><td>TERMINATED</td><td>127.0.0.1:9972 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00052</td><td>TERMINATED</td><td>127.0.0.1:896  </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00044</td><td>TERMINATED</td><td>127.0.0.1:14040</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00054</td><td>TERMINATED</td><td>127.0.0.1:14636</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.4555</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00042</td><td>TERMINATED</td><td>127.0.0.1:11516</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00062</td><td>TERMINATED</td><td>127.0.0.1:16480</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -1     </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00061</td><td>TERMINATED</td><td>127.0.0.1:8268 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -1     </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00059</td><td>TERMINATED</td><td>127.0.0.1:11468</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.4555</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00048</td><td>TERMINATED</td><td>127.0.0.1:5404 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.0694</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00046</td><td>TERMINATED</td><td>127.0.0.1:4724 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.0991</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00056</td><td>TERMINATED</td><td>127.0.0.1:4076 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -1     </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00057</td><td>TERMINATED</td><td>127.0.0.1:15764</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.01</td><td style=\"text-align: right;\">              -0.98  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00045</td><td>TERMINATED</td><td>127.0.0.1:16864</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.01</td><td style=\"text-align: right;\">              -0.8513</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00053</td><td>TERMINATED</td><td>127.0.0.1:9820 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -1     </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00041</td><td>TERMINATED</td><td>127.0.0.1:1616 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.03</td><td style=\"text-align: right;\">              -0.8212</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00051</td><td>TERMINATED</td><td>127.0.0.1:8788 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.7822</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00033</td><td>TERMINATED</td><td>127.0.0.1:2032 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00047</td><td>TERMINATED</td><td>127.0.0.1:8684 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -1     </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00039</td><td>TERMINATED</td><td>127.0.0.1:13464</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.11</td><td style=\"text-align: right;\">              -0.78  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00036</td><td>TERMINATED</td><td>127.0.0.1:5652 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.1882</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00038</td><td>TERMINATED</td><td>127.0.0.1:2124 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.01</td><td style=\"text-align: right;\">              -0.98  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00037</td><td>TERMINATED</td><td>127.0.0.1:14836</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.01</td><td style=\"text-align: right;\">              -0.98  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00026</td><td>TERMINATED</td><td>127.0.0.1:16368</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00031</td><td>TERMINATED</td><td>127.0.0.1:13532</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.0298</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00028</td><td>TERMINATED</td><td>127.0.0.1:13548</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00035</td><td>TERMINATED</td><td>127.0.0.1:13068</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.0199</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00032</td><td>TERMINATED</td><td>127.0.0.1:9608 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -1     </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00043</td><td>TERMINATED</td><td>127.0.0.1:9604 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.06</td><td style=\"text-align: right;\">              -0.88  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00023</td><td>TERMINATED</td><td>127.0.0.1:9192 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00034</td><td>TERMINATED</td><td>127.0.0.1:13984</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -1     </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00050</td><td>TERMINATED</td><td>127.0.0.1:12860</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -1     </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00024</td><td>TERMINATED</td><td>127.0.0.1:8048 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.7327</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00016</td><td>TERMINATED</td><td>127.0.0.1:260  </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.01</td><td style=\"text-align: right;\">              -0.9602</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00040</td><td>TERMINATED</td><td>127.0.0.1:8992 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.1189</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00021</td><td>TERMINATED</td><td>127.0.0.1:11888</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00030</td><td>TERMINATED</td><td>127.0.0.1:8720 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.04</td><td style=\"text-align: right;\">              -0.92  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00029</td><td>TERMINATED</td><td>127.0.0.1:2616 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.11</td><td style=\"text-align: right;\">              -0.78  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00006</td><td>TERMINATED</td><td>127.0.0.1:3340 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00005</td><td>TERMINATED</td><td>127.0.0.1:5268 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00012</td><td>TERMINATED</td><td>127.0.0.1:13156</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00027</td><td>TERMINATED</td><td>127.0.0.1:7264 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00004</td><td>TERMINATED</td><td>127.0.0.1:14852</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00019</td><td>TERMINATED</td><td>127.0.0.1:5996 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.01  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00020</td><td>TERMINATED</td><td>127.0.0.1:1500 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.2674</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00022</td><td>TERMINATED</td><td>127.0.0.1:9796 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.9109</td><td style=\"text-align: right;\">  15000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00014</td><td>TERMINATED</td><td>127.0.0.1:14992</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.01</td><td style=\"text-align: right;\">              -0.98  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00025</td><td>TERMINATED</td><td>127.0.0.1:5532 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -1     </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00018</td><td>TERMINATED</td><td>127.0.0.1:10212</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.05</td><td style=\"text-align: right;\">              -0.9   </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00002</td><td>TERMINATED</td><td>127.0.0.1:5892 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.36</td><td style=\"text-align: right;\">              -0.2602</td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00013</td><td>TERMINATED</td><td>127.0.0.1:2092 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.03</td><td style=\"text-align: right;\">              -0.94  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00017</td><td>TERMINATED</td><td>127.0.0.1:7956 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.9802</td><td style=\"text-align: right;\">  15000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00015</td><td>TERMINATED</td><td>127.0.0.1:5176 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -1     </td><td style=\"text-align: right;\">  15000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00001</td><td>TERMINATED</td><td>127.0.0.1:2360 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.4951</td><td style=\"text-align: right;\">  15000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00008</td><td>TERMINATED</td><td>127.0.0.1:5864 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.05</td><td style=\"text-align: right;\">              -0.9   </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00011</td><td>TERMINATED</td><td>127.0.0.1:3332 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.7327</td><td style=\"text-align: right;\">  15000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00010</td><td>TERMINATED</td><td>127.0.0.1:13180</td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.01</td><td style=\"text-align: right;\">              -0.98  </td><td style=\"text-align: right;\">  15000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00000</td><td>TERMINATED</td><td>127.0.0.1:3452 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0.09</td><td style=\"text-align: right;\">              -0.82  </td><td style=\"text-align: right;\">  30000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00003</td><td>TERMINATED</td><td>127.0.0.1:9328 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -0.9901</td><td style=\"text-align: right;\">  15000</td></tr>\n",
              "<tr><td>DEFAULT_747b2_00009</td><td>TERMINATED</td><td>127.0.0.1:6292 </td><td>       </td><td>False       </td><td style=\"text-align: right;\">       0   </td><td style=\"text-align: right;\">              -1     </td><td style=\"text-align: right;\">  15000</td></tr>\n",
              "</tbody>\n",
              "</table><br><br>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "2022-05-26 04:24:03,508\tINFO tune.py:626 -- Total run time: 21479.86 seconds (21479.32 seconds for the tuning loop).\n"
          ]
        }
      ],
      "source": [
        "result = tune.run(\n",
        "    partial(perform_experiment),\n",
        "    resources_per_trial={\n",
        "        \"cpu\": 6/parallel_runs, \n",
        "        \"gpu\": 1/parallel_runs},\n",
        "    config=config,\n",
        "    num_samples=150,\n",
        "    scheduler=scheduler,\n",
        "    progress_reporter=reporter,\n",
        "    name='full_run',\n",
        "    log_to_file=True,\n",
        "    verbose=2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>if_trained</th>\n",
              "      <th>win_ratio</th>\n",
              "      <th>episode_reward_mean</th>\n",
              "      <th>steps</th>\n",
              "      <th>config/gamma</th>\n",
              "      <th>config/lr</th>\n",
              "      <th>config/n_hidden_layers</th>\n",
              "      <th>config/random_scaling</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>True</td>\n",
              "      <td>0.71</td>\n",
              "      <td>0.4200</td>\n",
              "      <td>5081</td>\n",
              "      <td>0.929049</td>\n",
              "      <td>0.008325</td>\n",
              "      <td>1</td>\n",
              "      <td>0.999778</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>66</th>\n",
              "      <td>False</td>\n",
              "      <td>0.41</td>\n",
              "      <td>-0.1800</td>\n",
              "      <td>30000</td>\n",
              "      <td>0.267731</td>\n",
              "      <td>0.000036</td>\n",
              "      <td>1</td>\n",
              "      <td>0.999620</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>83</th>\n",
              "      <td>False</td>\n",
              "      <td>0.40</td>\n",
              "      <td>-0.2000</td>\n",
              "      <td>30000</td>\n",
              "      <td>0.285017</td>\n",
              "      <td>0.001000</td>\n",
              "      <td>1</td>\n",
              "      <td>0.999745</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>139</th>\n",
              "      <td>False</td>\n",
              "      <td>0.36</td>\n",
              "      <td>-0.2602</td>\n",
              "      <td>30000</td>\n",
              "      <td>0.641106</td>\n",
              "      <td>0.000013</td>\n",
              "      <td>1</td>\n",
              "      <td>0.999630</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>74</th>\n",
              "      <td>False</td>\n",
              "      <td>0.28</td>\n",
              "      <td>-0.4400</td>\n",
              "      <td>30000</td>\n",
              "      <td>0.316274</td>\n",
              "      <td>0.000480</td>\n",
              "      <td>1</td>\n",
              "      <td>0.999928</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69</th>\n",
              "      <td>False</td>\n",
              "      <td>0.16</td>\n",
              "      <td>-0.6800</td>\n",
              "      <td>30000</td>\n",
              "      <td>0.156376</td>\n",
              "      <td>0.000060</td>\n",
              "      <td>1</td>\n",
              "      <td>0.999633</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>68</th>\n",
              "      <td>False</td>\n",
              "      <td>0.15</td>\n",
              "      <td>-0.7000</td>\n",
              "      <td>30000</td>\n",
              "      <td>0.193105</td>\n",
              "      <td>0.000463</td>\n",
              "      <td>1</td>\n",
              "      <td>0.999901</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>127</th>\n",
              "      <td>False</td>\n",
              "      <td>0.11</td>\n",
              "      <td>-0.7800</td>\n",
              "      <td>30000</td>\n",
              "      <td>0.241511</td>\n",
              "      <td>0.014428</td>\n",
              "      <td>2</td>\n",
              "      <td>0.999710</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>109</th>\n",
              "      <td>False</td>\n",
              "      <td>0.11</td>\n",
              "      <td>-0.7800</td>\n",
              "      <td>30000</td>\n",
              "      <td>0.253586</td>\n",
              "      <td>0.000028</td>\n",
              "      <td>1</td>\n",
              "      <td>0.999790</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>147</th>\n",
              "      <td>False</td>\n",
              "      <td>0.09</td>\n",
              "      <td>-0.8200</td>\n",
              "      <td>30000</td>\n",
              "      <td>0.156308</td>\n",
              "      <td>0.018064</td>\n",
              "      <td>2</td>\n",
              "      <td>0.999751</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>84</th>\n",
              "      <td>False</td>\n",
              "      <td>0.08</td>\n",
              "      <td>-0.8400</td>\n",
              "      <td>30000</td>\n",
              "      <td>0.100984</td>\n",
              "      <td>0.000107</td>\n",
              "      <td>2</td>\n",
              "      <td>0.999536</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>118</th>\n",
              "      <td>False</td>\n",
              "      <td>0.06</td>\n",
              "      <td>-0.8800</td>\n",
              "      <td>30000</td>\n",
              "      <td>0.111315</td>\n",
              "      <td>0.000149</td>\n",
              "      <td>1</td>\n",
              "      <td>0.999646</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>144</th>\n",
              "      <td>False</td>\n",
              "      <td>0.05</td>\n",
              "      <td>-0.9000</td>\n",
              "      <td>30000</td>\n",
              "      <td>0.117661</td>\n",
              "      <td>0.000020</td>\n",
              "      <td>2</td>\n",
              "      <td>0.999770</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>138</th>\n",
              "      <td>False</td>\n",
              "      <td>0.05</td>\n",
              "      <td>-0.9000</td>\n",
              "      <td>30000</td>\n",
              "      <td>0.363153</td>\n",
              "      <td>0.000051</td>\n",
              "      <td>1</td>\n",
              "      <td>0.999764</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>False</td>\n",
              "      <td>0.04</td>\n",
              "      <td>-0.9200</td>\n",
              "      <td>30000</td>\n",
              "      <td>0.247895</td>\n",
              "      <td>0.000244</td>\n",
              "      <td>1</td>\n",
              "      <td>0.999698</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>126</th>\n",
              "      <td>False</td>\n",
              "      <td>0.04</td>\n",
              "      <td>-0.9200</td>\n",
              "      <td>30000</td>\n",
              "      <td>0.199897</td>\n",
              "      <td>0.000625</td>\n",
              "      <td>2</td>\n",
              "      <td>0.999530</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>140</th>\n",
              "      <td>False</td>\n",
              "      <td>0.03</td>\n",
              "      <td>-0.9400</td>\n",
              "      <td>30000</td>\n",
              "      <td>0.178834</td>\n",
              "      <td>0.001820</td>\n",
              "      <td>1</td>\n",
              "      <td>0.999941</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>73</th>\n",
              "      <td>False</td>\n",
              "      <td>0.03</td>\n",
              "      <td>-0.9400</td>\n",
              "      <td>30000</td>\n",
              "      <td>0.154656</td>\n",
              "      <td>0.000099</td>\n",
              "      <td>2</td>\n",
              "      <td>0.999920</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>51</th>\n",
              "      <td>False</td>\n",
              "      <td>0.03</td>\n",
              "      <td>-0.9400</td>\n",
              "      <td>30000</td>\n",
              "      <td>0.152964</td>\n",
              "      <td>0.004488</td>\n",
              "      <td>2</td>\n",
              "      <td>0.999589</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>105</th>\n",
              "      <td>False</td>\n",
              "      <td>0.03</td>\n",
              "      <td>-0.8212</td>\n",
              "      <td>30000</td>\n",
              "      <td>0.269835</td>\n",
              "      <td>0.009310</td>\n",
              "      <td>2</td>\n",
              "      <td>0.999768</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     if_trained  win_ratio  episode_reward_mean  steps  config/gamma  \\\n",
              "35         True       0.71               0.4200   5081      0.929049   \n",
              "66        False       0.41              -0.1800  30000      0.267731   \n",
              "83        False       0.40              -0.2000  30000      0.285017   \n",
              "139       False       0.36              -0.2602  30000      0.641106   \n",
              "74        False       0.28              -0.4400  30000      0.316274   \n",
              "69        False       0.16              -0.6800  30000      0.156376   \n",
              "68        False       0.15              -0.7000  30000      0.193105   \n",
              "127       False       0.11              -0.7800  30000      0.241511   \n",
              "109       False       0.11              -0.7800  30000      0.253586   \n",
              "147       False       0.09              -0.8200  30000      0.156308   \n",
              "84        False       0.08              -0.8400  30000      0.100984   \n",
              "118       False       0.06              -0.8800  30000      0.111315   \n",
              "144       False       0.05              -0.9000  30000      0.117661   \n",
              "138       False       0.05              -0.9000  30000      0.363153   \n",
              "29        False       0.04              -0.9200  30000      0.247895   \n",
              "126       False       0.04              -0.9200  30000      0.199897   \n",
              "140       False       0.03              -0.9400  30000      0.178834   \n",
              "73        False       0.03              -0.9400  30000      0.154656   \n",
              "51        False       0.03              -0.9400  30000      0.152964   \n",
              "105       False       0.03              -0.8212  30000      0.269835   \n",
              "\n",
              "     config/lr  config/n_hidden_layers  config/random_scaling  \n",
              "35    0.008325                       1               0.999778  \n",
              "66    0.000036                       1               0.999620  \n",
              "83    0.001000                       1               0.999745  \n",
              "139   0.000013                       1               0.999630  \n",
              "74    0.000480                       1               0.999928  \n",
              "69    0.000060                       1               0.999633  \n",
              "68    0.000463                       1               0.999901  \n",
              "127   0.014428                       2               0.999710  \n",
              "109   0.000028                       1               0.999790  \n",
              "147   0.018064                       2               0.999751  \n",
              "84    0.000107                       2               0.999536  \n",
              "118   0.000149                       1               0.999646  \n",
              "144   0.000020                       2               0.999770  \n",
              "138   0.000051                       1               0.999764  \n",
              "29    0.000244                       1               0.999698  \n",
              "126   0.000625                       2               0.999530  \n",
              "140   0.001820                       1               0.999941  \n",
              "73    0.000099                       2               0.999920  \n",
              "51    0.004488                       2               0.999589  \n",
              "105   0.009310                       2               0.999768  "
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "res_df = result.dataframe()\n",
        "res_df = res_df[['if_trained', 'win_ratio', 'episode_reward_mean', 'steps','config/gamma',\n",
        "       'config/lr', 'config/n_hidden_layers', 'config/random_scaling']]\n",
        "\n",
        "res_df.sort_values(['win_ratio'], ascending=False)[:20]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZixMrby7Y6CG"
      },
      "source": [
        "## Save results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "base_results_df_path = os.path.join(\"..\", \"results\", \"auto_hp_tuning\")\n",
        "trial_num = 0\n",
        "df_saved = False\n",
        "\n",
        "while not df_saved:\n",
        "  results_df_path = os.path.join(base_results_df_path, \"results_\"+str(trial_num)+\".csv\")\n",
        "  if os.path.exists(results_df_path):\n",
        "    trial_num+=1\n",
        "  else:\n",
        "    res_df.to_csv(results_df_path)\n",
        "    df_saved = True"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "DQL_quant_sim_hypreparameter_finetuning.ipynb",
      "provenance": []
    },
    "interpreter": {
      "hash": "1e78155a945478b8f958a676071cae883637dc0ce737ab4b9d1ebabad5503739"
    },
    "kernelspec": {
      "display_name": "Python 3.9.11 ('qTorchGPUauto')",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.11"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}