src/demo/examples/sample_HiveNAS.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**SOURCE:** https://github.com/ThunderStruct/HiveNAS/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XSKHSgObwIUq"
   },
   "source": [
    "### Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "luHpeKqmwNNZ"
   },
   "outputs": [],
   "source": [
    "!ln -sf /opt/bin/nvidia-smi /usr/bin/nvidia-smi\n",
    "!pip install gputil\n",
    "!pip install psutil\n",
    "!pip install humanize\n",
    "!pip install pyyaml\n",
    "\n",
    "import os\n",
    "import re\n",
    "import gc\n",
    "import sys\n",
    "import time\n",
    "import yaml\n",
    "import errno\n",
    "import hashlib\n",
    "import base64\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import networkx as nx\n",
    "from abc import ABC, abstractmethod\n",
    "from IPython.display import display, clear_output\n",
    "from ipywidgets import widgets\n",
    "from tensorflow.test import gpu_device_name\n",
    "from tensorflow.image import random_contrast, random_saturation\n",
    "from tensorflow.keras import backend as K\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "from tensorflow.keras.models import Sequential, Model, load_model\n",
    "from tensorflow.keras.optimizers import RMSprop, Adam\n",
    "from tensorflow.keras.layers import Input, Activation, Conv2D, Flatten, MaxPooling2D\n",
    "from tensorflow.keras.layers import SeparableConv2D, concatenate, add, Dense, Dropout\n",
    "from tensorflow.keras.layers import AveragePooling2D, BatchNormalization, ReLU, Add\n",
    "from tensorflow.keras.datasets import cifar10, mnist, fashion_mnist\n",
    "from tensorflow.keras.callbacks import Callback\n",
    "from tensorflow.keras import utils\n",
    "from matplotlib import pyplot as plt\n",
    "from scipy import optimize\n",
    "from copy import deepcopy\n",
    "from functools import partial\n",
    "\n",
    "\n",
    "ON_COLAB = 'google.colab' in sys.modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "JWt0VuB_nI4O"
   },
   "outputs": [],
   "source": [
    "\"\"\" Preliminary configurations / tests\n",
    "\"\"\"\n",
    "\n",
    "def test_colab_GPU_mem():\n",
    "    '''Colab GPU memory test (Colab often shares a computing unit \\\n",
    "    with multiple users, ending up with less than 60% of the expected \\\n",
    "    GPU resources).\n",
    "\n",
    "    Restart instance to get reallocated resources.\n",
    "    '''\n",
    "\n",
    "    import psutil\n",
    "    import humanize\n",
    "    import GPUtil as GPU\n",
    "    GPUs = GPU.getGPUs()\n",
    "\n",
    "    gpu = GPUs[0]\n",
    "\n",
    "    process = psutil.Process(os.getpid())\n",
    "    print('Gen RAM Free: ' + humanize.naturalsize(psutil.virtual_memory().available), ' | Proc size: ' + humanize.naturalsize(process.memory_info().rss))\n",
    "    print('GPU RAM Free: {0:.0f}MB | Used: {1:.0f}MB | Util {2:3.0f}% | Total {3:.0f}MB'.format(gpu.memoryFree, gpu.memoryUsed, gpu.memoryUtil*100, gpu.memoryTotal))\n",
    "    # Check if GPU usage is > 2% (i.e device shared with other Colab users)\n",
    "    print('GPU Memory is shared! Restart the runtime.' if gpu.memoryUtil > 0.05 else 'GPU Memory is free!')\n",
    "\n",
    "\n",
    "if ON_COLAB:\n",
    "    from google.colab import files, drive #, output\n",
    "    # output.enable_custom_widget_manager()\n",
    "    drive.mount('/content/gdrive')\n",
    "\n",
    "\n",
    "    if gpu_device_name() != '/device:GPU:0':\n",
    "        print('GPU device not found -- Try enabling GPU acceleration in Colab\\'s runtime settings')\n",
    "\n",
    "    else:\n",
    "        print('GPU device found!')\n",
    "        test_colab_GPU_mem()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "23ljYpBl6G-p"
   },
   "source": [
    "### Helper Tools"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "txwH2M_YqiTt"
   },
   "source": [
    "#### Prompt Handler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "7sKcShxukoBW"
   },
   "outputs": [],
   "source": [
    "\"\"\"User prompts-handler\n",
    "\"\"\"\n",
    "\n",
    "class PromptHandler:\n",
    "    '''Wrapper for input prompt-handling methods\n",
    "    '''\n",
    "\n",
    "    @staticmethod\n",
    "    def setup_ipy_widgets(on_yaml_export, on_yaml_init):\n",
    "        ''' Sets up configuration loader/exporter IPython widgets \\\n",
    "        (Google Colab-exclusive).\n",
    "\n",
    "        Args:\n",
    "            on_yaml_export (func): action to execute upon \\\n",
    "            yaml export button click\n",
    "            on_yaml_init (func): action to execute upon yaml \\\n",
    "            load button click\n",
    "        '''\n",
    "\n",
    "        path_textarea = widgets.Text(placeholder='/path/to/config.yaml',\n",
    "                                     value='/content/config.yaml')\n",
    "        export_btn = widgets.Button(description=\"Export *Form* Config\")\n",
    "        load_btn = widgets.Button(description=\"Load Config\")\n",
    "        output = widgets.Output()\n",
    "\n",
    "        def on_export_click(b):\n",
    "            path = path_textarea.value\n",
    "\n",
    "            filename = os.path.basename(path)\n",
    "            path = os.path.dirname(path)\n",
    "\n",
    "            with output:\n",
    "                clear_output(wait=True)\n",
    "                if filename == '':\n",
    "                    print('\\nInvalid path / filename!\\n\\n')\n",
    "                on_yaml_export(path, filename)\n",
    "\n",
    "        def on_load_click(b):\n",
    "            path = path_textarea.value\n",
    "\n",
    "            with output:\n",
    "                clear_output(wait=True)\n",
    "                on_yaml_init(path)\n",
    "\n",
    "\n",
    "        export_btn.on_click(on_export_click)\n",
    "        load_btn.on_click(on_load_click)\n",
    "\n",
    "        bot_box = widgets.HBox([load_btn, export_btn])\n",
    "        cont_box = widgets.VBox([path_textarea, bot_box])\n",
    "\n",
    "        display(cont_box, output)\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def prompt_yes_no(question, default='y'):\n",
    "        '''Yes/no query; reverts to default value if no input is given\n",
    "\n",
    "        Args:\n",
    "            question (str): printed prompt question\n",
    "            default (str, optional): user answer to revert to if no \\\n",
    "            response is given (empty input) ; defaults to \"yes\"\n",
    "\n",
    "        Returns:\n",
    "            bool: user response\n",
    "        '''\n",
    "\n",
    "        valid_res = {\n",
    "            'yes': True,\n",
    "            'y': True,\n",
    "            'no': False,\n",
    "            'n': False\n",
    "        }\n",
    "\n",
    "        choice = None\n",
    "\n",
    "        while choice not in valid_res:\n",
    "            choice = input(f'{question} (y/n): ').lower().replace(' ', '') or default\n",
    "\n",
    "        return valid_res[choice]\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3HHRRmRhotHR"
   },
   "source": [
    "#### File Handler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "t7GSEuUZoyPB"
   },
   "outputs": [],
   "source": [
    "\"\"\"HiveNAS file-handling methods\n",
    "\"\"\"\n",
    "\n",
    "class FileHandler:\n",
    "    '''Wrapper for file-handling methods\n",
    "    '''\n",
    "\n",
    "    __VALID_PATHS = {}\n",
    "\n",
    "    @staticmethod\n",
    "    def __path_exists(path):\n",
    "        '''Checks if file exists\n",
    "\n",
    "        Args:\n",
    "            path (str): path to file (includes filename and extension)\n",
    "\n",
    "        Returns:\n",
    "            bool: whether or not the file exists\n",
    "        '''\n",
    "\n",
    "        return os.path.exists(path)\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def validate_path(path):\n",
    "        '''Ensures that a given directory path is universaly valid \\\n",
    "        (Windows/Linux/MacOS/POSIX) and creates it.\n",
    "\n",
    "        Prompts user for overwriting (using :class:`~utils.prompt_handler.PromptHandler`) \\\n",
    "        if it already exists\n",
    "\n",
    "        Args:\n",
    "            path (str): path to validated\n",
    "\n",
    "        Returns:\n",
    "            bool: validity of the given path\n",
    "        '''\n",
    "\n",
    "        # Directory already exists, prompt for overwrite permission (first time only)\n",
    "        if path not in FileHandler.__VALID_PATHS and FileHandler.__path_exists(path):\n",
    "\n",
    "            FileHandler.__VALID_PATHS[path] = True\n",
    "\n",
    "            if len(os.listdir(path)) == 0:\n",
    "                # directory exists and is empty -> is valid\n",
    "                return FileHandler.__VALID_PATHS[path]\n",
    "\n",
    "            # directory exists and is NOT empty\n",
    "            FileHandler.__VALID_PATHS[path] = True\n",
    "            print(f'\\nPath ({path}) already exists!\\n\\n')\n",
    "            return PromptHandler.prompt_yes_no('Would you like to overwrite files in this path?')\n",
    "\n",
    "        # Previously evaluated\n",
    "        if path in FileHandler.__VALID_PATHS:\n",
    "            return FileHandler.__VALID_PATHS[path]\n",
    "\n",
    "        # Check the validity of the given path\n",
    "        try:\n",
    "            os.makedirs(path)\n",
    "            FileHandler.__VALID_PATHS[path] = True\n",
    "        except OSError as e:\n",
    "            # path invalid\n",
    "            print('\\nBase path and/or config version are invalid! Please choose path-friendly names.\\n')\n",
    "            FileHandler.__VALID_PATHS[path] = False\n",
    "\n",
    "        return FileHandler.__VALID_PATHS[path]\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def create_dir(path):\n",
    "        '''Recursively creates new directory if it does not exist\n",
    "\n",
    "        Args:\n",
    "            path (str): directory path to be created\n",
    "        '''\n",
    "\n",
    "        if not FileHandler.__path_exists(path):\n",
    "            os.makedirs(path)\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def path_must_exist(path):\n",
    "        '''Checks if file exists and raises error if it is not.\n",
    "        Used when the logic of the algorithm depends on the loaded file\n",
    "\n",
    "        Args:\n",
    "            path (str): path to file\n",
    "\n",
    "        Raises:\n",
    "            :class:`FileNotFoundError`: file does not exist\n",
    "        '''\n",
    "\n",
    "        if not FileHandler.__path_exists(path):\n",
    "            raise FileNotFoundError(errno.ENOENT,\n",
    "                                    os.strerror(errno.ENOENT),\n",
    "                                    path)\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def save_pickle(p_dict, path, filename, force_dir=True):\n",
    "        '''Saves the given dictionary as a :class:`pickle`\n",
    "\n",
    "        Args:\n",
    "            p_dict (dict): data to be saved\n",
    "            path (str): path to save directory\n",
    "            filename (str): output filename\n",
    "            force_dir (bool, optional): whether or not to force create \\\n",
    "            the directory if it does not exist\n",
    "\n",
    "        Returns:\n",
    "            bool: save operation status\n",
    "        '''\n",
    "\n",
    "        if not FileHandler.__path_exists(path):\n",
    "            if force_dir:\n",
    "                FileHandler.create_dir(path)\n",
    "            else:\n",
    "                # directory does not exist and cannot create dir\n",
    "                return False\n",
    "\n",
    "        # dump pickle\n",
    "        with open(os.path.join(path, filename), 'wb') as handle:\n",
    "            pickle.dump(p_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "        return True\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def load_pickle(path, default_dict={}):\n",
    "        '''Loads :class:`pickle`  and returns decoded dictionary\n",
    "\n",
    "        Args:\n",
    "            path (str): path to :class:`pickle` file (includes filename)\n",
    "            default_dict (dict, optional): default dictionary to return \\\n",
    "            if the pickle does not exist (defaults to :code:`{ }`)\n",
    "\n",
    "        Returns:\n",
    "            dict: loaded data\n",
    "        '''\n",
    "\n",
    "        res = default_dict\n",
    "\n",
    "        if FileHandler.__path_exists(path):\n",
    "            with open(path, 'rb') as handle:\n",
    "                res = pickle.load(handle)\n",
    "\n",
    "        return res\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def save_df(df, path, filename, force_dir=True):\n",
    "        '''Saves a Pandas DataFrame to the given path\n",
    "\n",
    "        Args:\n",
    "            df (:class:`pandas.DataFrame`): dataframe to be saved\n",
    "            path (str): save directory path\n",
    "            filename (str): output filename\n",
    "            force_dir (bool, optional): whether or not to force create \\\n",
    "            the directory if it does not exist\n",
    "\n",
    "        Returns:\n",
    "            bool: save operation status\n",
    "        '''\n",
    "\n",
    "        if not FileHandler.__path_exists(path):\n",
    "            if force_dir:\n",
    "                FileHandler.create_dir(path)\n",
    "            else:\n",
    "                # directory does not exist and cannot create dir\n",
    "                return False\n",
    "\n",
    "        # save dataframe\n",
    "        df.to_csv(os.path.join(path, filename))\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def load_df(path, default_df=None):\n",
    "        '''Loads a :class:`pandas.DataFrame`\n",
    "\n",
    "        Args:\n",
    "            path (str): path to the dataframe\n",
    "            default_df (None, optional): default dictionary to return \\\n",
    "            if the dataframe does not exist (defaults to empty dataframe)\n",
    "\n",
    "        Returns:\n",
    "            :class:`pandas.DataFrame`: loaded dataframe or default\n",
    "        '''\n",
    "\n",
    "        res = default_df or pd.DataFrame()\n",
    "\n",
    "        if FileHandler.__path_exists(path):\n",
    "            res = pd.read_csv(path, header=0, index_col=0)\n",
    "\n",
    "        return res\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def export_yaml(config_dict, path, filename, file_version_comment='', force_dir=True):\n",
    "        '''Exports a given dictionary to a yaml file\n",
    "\n",
    "        Args:\n",
    "            config_dict (dict): dictionary to be saved as yaml\n",
    "            path (str): save directory path\n",
    "            filename (str): output filename\n",
    "            file_version_comment (str, optional): optional string to be prepended at /\n",
    "            the top of the yaml file as a comment (typically used to highlight the /\n",
    "            configuration version)\n",
    "            force_dir (bool, optional): whether or not to force create \\\n",
    "            the directory if it does not exist\n",
    "\n",
    "        Returns:\n",
    "            bool: save operation status\n",
    "        '''\n",
    "\n",
    "        if not FileHandler.__path_exists(path):\n",
    "            if force_dir:\n",
    "                FileHandler.create_dir(path)\n",
    "            else:\n",
    "                return False\n",
    "\n",
    "        with open(os.path.join(path, filename), 'w') as handler:\n",
    "            if file_version_comment:\n",
    "                handler.write(f'\\n# {file_version_comment}\\n\\n')\n",
    "            yaml.dump(config_dict, handler, default_flow_style=False)\n",
    "\n",
    "        return True\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def load_yaml(path, loader, default_dict={}):\n",
    "        '''Loads a yaml config file and returns it as dict\n",
    "\n",
    "        Args:\n",
    "            path (str): path to the yaml file (includes filename)\n",
    "            loader (:class:`yaml.SafeLoader`): yaml custom loader defined \\\n",
    "            in :func:`~config.params.Params.export_yaml`\n",
    "            default_dict (dict, optional): default dictionary to return \\\n",
    "            if the yaml file does not exist (defaults to :code:`{ }`)\n",
    "\n",
    "        Returns:\n",
    "            dict: loaded yaml data\n",
    "        '''\n",
    "\n",
    "        res = default_dict\n",
    "\n",
    "        if FileHandler.__path_exists(path):\n",
    "            with open(path, 'r') as handler:\n",
    "                res = yaml.load(handler, Loader=loader)\n",
    "\n",
    "        return res\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6pOEN80_BQN1"
   },
   "source": [
    "#### Operational Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": "code",
    "id": "XaMsCKlzBYyi"
   },
   "outputs": [],
   "source": [
    "\"\"\"All operational parameters used by HiveNAS and configuration methods\n",
    "\"\"\"\n",
    "\n",
    "class Params:\n",
    "    '''Wrapper for all global operational parameters\n",
    "    and the configuration loader\n",
    "    '''\n",
    "\n",
    "    @staticmethod\n",
    "    def config_form():\n",
    "        '''Facilitates the configuration UI form (for Google Colab) \\\n",
    "        and exports all parameters as a dictionary\n",
    "\n",
    "        Returns:\n",
    "            dict: main global parameters dictionary (locals)\n",
    "        '''\n",
    "\n",
    "\n",
    "        ''' Configuration Version (used as filenames) '''\n",
    "        CONFIG_VERSION = 'hivenas_v2_res'   #@param {type:\"string\"}\n",
    "\n",
    "\n",
    "        #@markdown ## ABC Optimizer Parameters\n",
    "        #@markdown ---\n",
    "\n",
    "\n",
    "        ''' Optimization problem (NAS or Numerical Benchmarks to test ABC) '''\n",
    "        OPTIMIZATION_OBJECTIVE = 'NAS'  #@param ['NAS', 'Sphere_max', 'Sphere_min', 'Rosenbrock']\n",
    "\n",
    "        ''' Max trials per Scout (i.e initial Food Source) '''\n",
    "        ABANDONMENT_LIMIT = 3  #@param {type:\"slider\", min:1, max:50, step:1}\n",
    "\n",
    "        ''' Number of bees in the colony (Employees + Onlookers) '''\n",
    "        COLONY_SIZE = 7    #@param {type:\"slider\", min:1, max:50, step:1}\n",
    "\n",
    "        ''' Distribution of Employees to Onlookers, resulting number of EmployeeBees = # of ScoutBees '''\n",
    "        EMPLOYEE_ONLOOKER_RATIO = 0.43   #@param {type:\"slider\", min:0.1, max:1.0, step:0.05}\n",
    "\n",
    "        ''' Number of ABC optimization iterations '''\n",
    "        ITERATIONS_COUNT = 12    #@param {type:\"slider\", min:1, max:100, step:1}\n",
    "\n",
    "\n",
    "        #@markdown \\\n",
    "        #@markdown ## File-Handling Parameters\n",
    "        #@markdown ---\n",
    "\n",
    "\n",
    "        ''' Save results every N evaluations (not iterations; iterations * colony_size) '''\n",
    "        RESULTS_SAVE_FREQUENCY = 1   #@param {type:\"slider\", min:1, max:100, step:1}\n",
    "\n",
    "        '''\n",
    "            Result files base path (path will be created if it does not exist)\n",
    "            A local folder will be created after the CONFIG_VERSION\n",
    "        '''\n",
    "        RESULTS_BASE_PATH = '/content/gdrive/MyDrive/PhD/HiveNAS/'  #@param {type:\"string\"}\n",
    "\n",
    "        ''' Training history files sub-path '''\n",
    "        HISTORY_FILES_SUBPATH = 'training_history/'     #@param {type:\"string\"}\n",
    "\n",
    "        ''' Enable weights saving for resumed training '''\n",
    "        ENABLE_WEIGHT_SAVING = False    #@param {type:\"boolean\"}\n",
    "\n",
    "        ''' Weight files sub-path (ensure that the path exists) '''\n",
    "        WEIGHT_FILES_SUBPATH = 'weights/'    #@param {type:\"string\"}\n",
    "\n",
    "        ''' Specifies whether or not to resume from existing results file (if exists)'''\n",
    "        RESUME_FROM_RESULTS_FILE = False   #@param {type:'boolean'}\n",
    "\n",
    "\n",
    "        #@markdown \\\n",
    "        #@markdown ## NAS Search Space Parameters\n",
    "        #@markdown ---\n",
    "\n",
    "\n",
    "        ''' -- NAS Search Space configuration -- '''\n",
    "\n",
    "        #@markdown *( layers & hyperparameters must be defined as partial functions in code )*\n",
    "\n",
    "        ''' Number of layers for sampled networks (excludes input/output stems) '''\n",
    "        DEPTH = 7     #@param {type:\"slider\", min:1, max:10, step:1}\n",
    "\n",
    "        ''' Search space operations '''\n",
    "        OPERATIONS = {\n",
    "            'search_space': {\n",
    "                'sep5x5_128': partial(SeparableConv2D, filters=128, kernel_size=(5,5), activation='relu', padding='same'),\n",
    "                'sep3x3_128': partial(SeparableConv2D, filters=128, kernel_size=(3,3), activation='relu', padding='same'),\n",
    "                'sep5x5_64': partial(SeparableConv2D, filters=64, kernel_size=(5,5), activation='relu', padding='same'),\n",
    "                'sep3x3_64': partial(SeparableConv2D, filters=64, kernel_size=(3,3), activation='relu', padding='same'),\n",
    "                'sep5x5_32': partial(SeparableConv2D, filters=32, kernel_size=(5,5), activation='relu', padding='same'),\n",
    "                'sep3x3_32': partial(SeparableConv2D, filters=32, kernel_size=(3,3), activation='relu', padding='same'),\n",
    "                'max_pool3x3': partial(MaxPooling2D, pool_size=(3,3), strides=(1,1), padding='same'),\n",
    "                'avg_pool3x3': partial(AveragePooling2D, pool_size=(3,3), strides=(1,1), padding='same'),\n",
    "                'dropout': partial(Dropout, rate=0.15),\n",
    "                'identity': partial(Activation, 'linear')\n",
    "            },\n",
    "            'reference_space': {\n",
    "                'conv3x3_16': partial(Conv2D, filters=16, kernel_size=(3,3), activation='relu'),\n",
    "                'sep5x5_128': partial(SeparableConv2D, filters=128, kernel_size=(5,5), activation='relu', padding='same'),\n",
    "                'sep3x3_128': partial(SeparableConv2D, filters=128, kernel_size=(3,3), activation='relu', padding='same'),\n",
    "                'sep5x5_64': partial(SeparableConv2D, filters=64, kernel_size=(5,5), activation='relu', padding='same'),\n",
    "                'sep3x3_64': partial(SeparableConv2D, filters=64, kernel_size=(3,3), activation='relu', padding='same'),\n",
    "                'sep5x5_32': partial(SeparableConv2D, filters=32, kernel_size=(5,5), activation='relu', padding='same'),\n",
    "                'sep3x3_32': partial(SeparableConv2D, filters=32, kernel_size=(3,3), activation='relu', padding='same'),\n",
    "                'max_pool3x3': partial(MaxPooling2D, pool_size=(3,3), strides=(1,1), padding='same'),\n",
    "                'avg_pool3x3': partial(AveragePooling2D, pool_size=(3,3), strides=(1,1), padding='same'),\n",
    "                'batch_norm': partial(BatchNormalization),\n",
    "                'dropout': partial(Dropout, rate=0.15),\n",
    "                'identity': partial(Activation, 'linear')\n",
    "            }\n",
    "        }\n",
    "\n",
    "        # Skip-Connections'/Residual Blocks' occurence rate (0.0 = disabled)\n",
    "        RESIDUAL_BLOCKS_RATE = 0.15    #@param {type:\"slider\", min:0.0, max:1.0, step:0.05}\n",
    "\n",
    "        CLASSICAL_RESNETS = True    #@param {type:\"boolean\"}\n",
    "\n",
    "        # Squeeze-Excitation blocks ratio\n",
    "        SE_RATIO = 0.0  #@param {type:\"slider\", min:0.0, max:0.25, step:0.25}\n",
    "\n",
    "\n",
    "        #@markdown \\\n",
    "        #@markdown ## NAS Evaluation Strategy Parameters\n",
    "        #@markdown ---\n",
    "\n",
    "\n",
    "        ''' -- NAS Evaluation Strategy configuration -- '''\n",
    "\n",
    "        ''' Dataset (classes/inputs are inferred internally) '''\n",
    "        DATASET = 'CIFAR10'   #@param [\"CIFAR10\", \"MNIST\", \"FASHION_MNIST\"]\n",
    "\n",
    "        ''' Static output stem, added to every candidate '''\n",
    "        OUTPUT_STEM = [\n",
    "            partial(Flatten),\n",
    "            partial(Dropout, rate=0.15),\n",
    "            partial(Dense, units=1024, activation='relu'),\n",
    "            partial(Dropout, rate=0.15),\n",
    "            partial(Dense, units=512, activation='relu')\n",
    "        ]\n",
    "\n",
    "        ''' Static input stem, added to every candidate '''\n",
    "        INPUT_STEM = [\n",
    "            # partial(Conv2D, filters=32, kernel_size=(3,3)),\n",
    "            # partial(BatchNormalization),\n",
    "            # partial(ReLU)\n",
    "        ]\n",
    "\n",
    "        ''' Epochs count per candidate network '''\n",
    "        EPOCHS = 5  #@param {type:\"slider\", min:1, max:25, step:1}\n",
    "\n",
    "        ''' Momentum Augmentation epochs (0 = disabled ; overrides ENABLE_WEIGHT_SAVING) '''\n",
    "        MOMENTUM_EPOCHS = 0 #@param {type:\"slider\", min:0, max:25}\n",
    "\n",
    "        ''' Epochs count for the best performing candidate upon full training '''\n",
    "        FULL_TRAIN_EPOCHS = 50 #@param {type:\"slider\", min:1, max:150, step:1}\n",
    "\n",
    "        '''\n",
    "            Threshold factor (beta) for early-stopping (refer to the TerminateOnThreshold class for details)\n",
    "                1.0 = all networks will be terminated (minimum accuracy = 100%)\n",
    "                0.0 = disable early-stopping, all networks will pass\n",
    "                0.25 = for 10 classes, val_acc > 0.325 at epoch 1 will not be terminated\n",
    "                       (tolerance decreased for every subsequent epoch)\n",
    "        '''\n",
    "        TERMINATION_THRESHOLD_FACTOR = 0.0 #@param {type:\"slider\", min:0.0, max:1.0, step:0.05}\n",
    "\n",
    "        ''' Diminishing factor (zeta) for termination threshold over epochs '''\n",
    "        TERMINATION_DIMINISHING_FACTOR = 0.25 #@param {type:\"slider\", min:0.1, max:1.0, step:0.05}\n",
    "\n",
    "        ''' Learning rate (overrides default optimizer lr) '''\n",
    "        LR = 0.001  #@param {type:\"slider\", min:0.001, max:0.1, step:0.001}\n",
    "\n",
    "        ''' Batch size for every candidate evaluation '''\n",
    "        BATCH_SIZE = 128     #@param {type:\"slider\", min:8, max:256, step:2}\n",
    "\n",
    "        ''' Optimizer used for both NAS and full-training methods '''\n",
    "        OPTIMIZER = 'Adam'    #@param [\"Adam\", \"RMSprop\"]\n",
    "\n",
    "\n",
    "        #@markdown \\\n",
    "        #@markdown ## Data Augmentation Parameters\n",
    "        #@markdown ---\n",
    "\n",
    "\n",
    "        '''\n",
    "            Enable affine transformations augmentation\n",
    "            (horizontal/vertical shifts, rotation, etc...)\n",
    "        '''\n",
    "\n",
    "        AFFINE_TRANSFORMATIONS_ENABLED = True   #@param {type:\"boolean\"}\n",
    "\n",
    "        ''' Probability of random cutout augmentation occurence (0.0 = disabled) '''\n",
    "        CUTOUT_PROB = 0.8    #@param {type:\"slider\", min:0.0, max:1.0, step:0.05}\n",
    "\n",
    "        ''' Probability of random saturation augmentation occurence (0.0 = disabled) '''\n",
    "        SATURATION_AUG_PROB = 0.75    #@param {type:\"slider\", min:0.0, max:1.0, step:0.05}\n",
    "\n",
    "        ''' Probability of random contrast augmentation occurence (0.0 = disabled) '''\n",
    "        CONTRAST_AUG_PROB = 0.75    #@param {type:\"slider\", min:0.0, max:1.0, step:0.05}\n",
    "\n",
    "        return locals()\n",
    "\n",
    "\n",
    "    ''' Main configuration dict '''\n",
    "    __CONFIG = config_form.__func__()\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def init_from_yaml(path):\n",
    "        '''Initializes the global parameters from a given yaml config file\n",
    "\n",
    "        Args:\n",
    "            path (str): path to yaml configuration file\n",
    "\n",
    "        '''\n",
    "\n",
    "\n",
    "        def param_op_constructor(loader: yaml.SafeLoader, node: yaml.nodes.MappingNode):\n",
    "            '''Constructs NAS Search Space operations (using the custom \\\n",
    "            yaml :code:`!Operation` tag)\n",
    "\n",
    "            Args:\n",
    "                loader (:class:`yaml.SafeLoader`): yaml default safe loader\n",
    "                node (:class:`yaml.nodes.MappingNode`): yaml mapping node\n",
    "\n",
    "            Returns:\n",
    "                :class:`functools.partial`: partial function containing the neural operation\n",
    "            '''\n",
    "\n",
    "            # constructs an operation partial function from yaml !Operation tags\n",
    "            op_dict = loader.construct_mapping(node)\n",
    "            op = op_dict['op']\n",
    "            del op_dict['op']\n",
    "\n",
    "            return partial(globals()[op], **op_dict)\n",
    "\n",
    "\n",
    "        def param_tuple_constructor(loader: yaml.SafeLoader, node: yaml.nodes.MappingNode):\n",
    "            '''Constructs a tuple from the standard \\\n",
    "            :code:`tag:yaml.org,2002:python/tuple` (:code:`!!python/tuple`) yaml tag\n",
    "\n",
    "            Args:\n",
    "                loader (:class:`yaml.SafeLoader`): yaml default safe loader\n",
    "                node (:class:`yaml.nodes.MappingNode`): yaml mapping node\n",
    "\n",
    "            Returns:\n",
    "                tuple: constructed tuple, \\\n",
    "                typically used to define kernel sizes/shapes in yaml\n",
    "            '''\n",
    "\n",
    "            # because for some reason we need an explicit tuple constructor\n",
    "\n",
    "            return tuple(loader.construct_sequence(node))\n",
    "\n",
    "        # register constructors\n",
    "        loader = yaml.SafeLoader\n",
    "        loader.add_constructor(u'tag:yaml.org,2002:python/tuple', param_tuple_constructor)\n",
    "        loader.add_constructor('!Operation', param_op_constructor)\n",
    "\n",
    "        config = FileHandler.load_yaml(path, loader)\n",
    "\n",
    "        if not config:\n",
    "            print(f'\\nConfig file ({path}) is either invalid or does not exist.\\n\\n')\n",
    "            return\n",
    "\n",
    "        for key,val in config.items():\n",
    "            if key not in Params.__CONFIG:\n",
    "                # ensure config file keys are valid and match the hard-coded template\n",
    "                print(f'\\nConfig file ({path}) is invalid. Skipping item ({key})... \\n\\n')\n",
    "                continue\n",
    "\n",
    "            Params.__CONFIG[key] = val\n",
    "\n",
    "        print(f'\\nSuccessfully loaded the operational parameters from {path}.\\n\\n')\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def export_yaml(path, filename, from_formdata=False):\n",
    "        '''Saves the current configurations to the given path as yaml\n",
    "\n",
    "        Args:\n",
    "            path (str): output path to save the yaml config file to\n",
    "            filename (str): output file name\n",
    "            from_formdata (bool, optional): determines whether the export instruction \\\n",
    "            originated from the Google Colab UI form or called in code. *When it originates \\\n",
    "            from the form, data reload is required to ensure consistency (could be \\\n",
    "            altered within the form)*\n",
    "        '''\n",
    "\n",
    "        def param_op_representer(dumper, data):\n",
    "            '''Serializes a partial function into the custom :code:`!Operation` \\\n",
    "            yaml tag\n",
    "\n",
    "            Args:\n",
    "                dumper (:class:`yaml.Dumper`): default pyyaml dumper\n",
    "                data (partial): partial function data to be serialized\n",
    "\n",
    "            Returns:\n",
    "                :class:`yaml.nodes.MappingNode`: yaml mapping node representing \\\n",
    "                the operation\n",
    "            '''\n",
    "\n",
    "            # serialize partial functions into yaml !Operation\n",
    "            serialized_data = {'op': data.func.__name__}\n",
    "            serialized_data.update(data.keywords)\n",
    "\n",
    "            return dumper.represent_mapping('!Operation', serialized_data, flow_style=True)\n",
    "\n",
    "        def param_tuple_representer(dumper, data):\n",
    "            '''Serializes a tuple into the :code:`tag:yaml.org,2002:python/tuple` \\\n",
    "            (:code:`!!python/tuple`) yaml tag\n",
    "\n",
    "            Args:\n",
    "                dumper (:class:`yaml.Dumper`): default pyyaml dumper\n",
    "                data (tuple): tuple data to be serialized\n",
    "\n",
    "            Returns:\n",
    "                :class:`yaml.nodes.MappingNode`: yaml mapping node representing \\\n",
    "                the tuple\n",
    "            '''\n",
    "\n",
    "            # serialize tuples into yaml !!python/tuple\n",
    "\n",
    "            return dumper.represent_sequence(u'tag:yaml.org,2002:python/tuple', data, flow_style=True)\n",
    "\n",
    "        # register representers\n",
    "        yaml.add_representer(tuple, param_tuple_representer)\n",
    "        yaml.add_representer(partial, param_op_representer)\n",
    "        yaml.Dumper.ignore_aliases = lambda *args : True\n",
    "\n",
    "        # data source (changing the Colab form does not reflect on the main dict)\n",
    "        data = Params.config_form() if from_formdata else Params.__CONFIG\n",
    "\n",
    "        if FileHandler.export_yaml(data,\n",
    "                                   path,\n",
    "                                   filename):\n",
    "            print(f'\\nConfiguration file saved successfully to ({os.path.join(path, filename)})!\\n\\n')\n",
    "        else:\n",
    "            print('\\nFailed to save config file!\\n\\n')\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def search_space_config():\n",
    "        '''Returns the search space config dict\n",
    "\n",
    "        Returns:\n",
    "            dict: dictionary containing the :class:`~core.nas.search_space.NASSearchSpace`-related \\\n",
    "            parameters\n",
    "        '''\n",
    "\n",
    "        res = {\n",
    "            'depth': Params['DEPTH'],\n",
    "            'operations': Params['OPERATIONS'],\n",
    "            'residual_blocks_rate': Params['RESIDUAL_BLOCKS_RATE']\n",
    "        }\n",
    "\n",
    "        return res\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def evaluation_strategy_config():\n",
    "        '''Returns the evaluation strategy config dict\n",
    "\n",
    "        Returns:\n",
    "            dict: dictionary containing the :class:`~core.nas.evaluation_strategy.NASEval`-related \\\n",
    "            parameters\n",
    "        '''\n",
    "\n",
    "        res = {\n",
    "            'dataset': Params['DATASET'],\n",
    "            'operations': Params['OPERATIONS'],\n",
    "            'output_stem': Params['OUTPUT_STEM'],\n",
    "            'input_stem': Params['INPUT_STEM'],\n",
    "            'epochs': Params['EPOCHS'],\n",
    "            'full_train_epochs': Params['FULL_TRAIN_EPOCHS'],\n",
    "            'lr': Params['LR'],\n",
    "            'batch_size': Params['BATCH_SIZE'],\n",
    "            'optimizer': globals()[Params['OPTIMIZER']],\n",
    "            'termination_threshold_factor': Params['TERMINATION_THRESHOLD_FACTOR'],\n",
    "            'termination_diminishing_factor': Params['TERMINATION_DIMINISHING_FACTOR'],\n",
    "            'momentum_epochs': Params['MOMENTUM_EPOCHS']\n",
    "        }\n",
    "\n",
    "        return res\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def get_results_path():\n",
    "        '''Gets the results path from :code:`RESULTS_BASE_PATH` and :code:`CONFIG_VERSION`\n",
    "\n",
    "        Returns:\n",
    "            str: the joined path to the results directory or :code:`None` if either \\\n",
    "            :code:`RESULTS_BASE_PATH` or :code:`CONFIG_VERSION` is invalid\n",
    "        '''\n",
    "\n",
    "        path = os.path.join(Params.__CONFIG['RESULTS_BASE_PATH'],\n",
    "                            f'{Params.__CONFIG[\"CONFIG_VERSION\"]}/')\n",
    "\n",
    "        if FileHandler.validate_path(path):\n",
    "            return path\n",
    "\n",
    "        return None\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def get_all_config():\n",
    "        '''Returns all operational parameters\n",
    "\n",
    "        Returns:\n",
    "            dict: returns the dict containing all configurations *(for \\\n",
    "            argparsing purposes)*\n",
    "        '''\n",
    "\n",
    "        return Params.__CONFIG\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def set_parameter(key, val):\n",
    "        '''Overrides a default parameter (used by argparser)\n",
    "\n",
    "        Args:\n",
    "            key (str): dictionary key to select parameter\n",
    "            val (any): new value to override default parameter\n",
    "        '''\n",
    "\n",
    "        if key not in Params.__CONFIG or not isinstance(val, type(Params.__CONFIG[key])):\n",
    "            # invalid key or value type\n",
    "            return False\n",
    "\n",
    "        Params.__CONFIG[key] = val\n",
    "\n",
    "        return True\n",
    "\n",
    "\n",
    "    def __class_getitem__(cls, key):\n",
    "        '''Subscript operator definition\n",
    "\n",
    "        *Static class subscripting :code:`__class_getitem__` requires Python 3.7+*\n",
    "\n",
    "        Used as :code:`Params['KEY']`\n",
    "\n",
    "        Args:\n",
    "            key (str): dictionary key to select parameter\n",
    "\n",
    "        Returns:\n",
    "            Any: subscripted parameter from the configuration dictionary\n",
    "        '''\n",
    "\n",
    "        return Params.__CONFIG[key]\n",
    "\n",
    "\n",
    "''' Init widgets if ran on Colab '''\n",
    "if ON_COLAB:\n",
    "    PromptHandler.setup_ipy_widgets(on_yaml_export=lambda p, f: Params.export_yaml(p, f, True),\n",
    "                                    on_yaml_init=lambda p: Params.init_from_yaml(p))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "E-DtNe9hv4p0"
   },
   "source": [
    "#### Logger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "PRTp4FdmwD3l"
   },
   "outputs": [],
   "source": [
    "\"\"\"Main event logging methods\n",
    "\"\"\"\n",
    "\n",
    "class Logger:\n",
    "    '''Wrapper for debug- and info-logging methods\n",
    "\n",
    "    Attributes:\n",
    "        EVALUATION_LOGGING (bool): determines whether to log evaluation data \\\n",
    "        *(toggle to manage logging clutter in case the data points' count is large \\\n",
    "        -- i.e Numerical Benchmarks, for instance)*\n",
    "    '''\n",
    "\n",
    "    __DEBUG_PREFIX  = 'DEBUG:'\n",
    "    __STATUS_PREFIX = 'STATUS:'\n",
    "    __EVAL_PREFIX = 'EVALUATION LOG:'\n",
    "    __MOMENTUM_PREFIX = 'MOMENTUM EVAL:'\n",
    "    __FILESAVE_PREFIX = 'FILE-SAVE SUCCESSFUL:'\n",
    "\n",
    "    __START_TIME = None\n",
    "\n",
    "    EVALUATION_LOGGING = False\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def debug(msg=None):\n",
    "        '''Debugging messages\n",
    "\n",
    "        Args:\n",
    "            msg (str, optional): debug message, defaults to \\\n",
    "            \"MARK\" to indicate whether the statement is reached\n",
    "        '''\n",
    "\n",
    "        print('{} {}'.format(Logger.__DEBUG_PREFIX,\n",
    "                            ('MARK' if msg is None else str(msg))))\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def status(itr, msg=None):\n",
    "        '''Generic logging\n",
    "\n",
    "        Args:\n",
    "            itr (int): current optimization iteration\n",
    "            msg (str, optional): status message, defaults to \\\n",
    "            \"MARK\" to indicate whether the statement is reached\n",
    "        '''\n",
    "\n",
    "        print('{} itr: {} -- {}'.format(Logger.__STATUS_PREFIX,\n",
    "                                        str(itr),\n",
    "                                        ('MARK' if msg is None else str(msg))))\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def evaluation_log(type, id, candidate_pos):\n",
    "        '''Logs pre-evaluation info for every candidate\n",
    "\n",
    "        Args:\n",
    "            type (str): bee type (Employee/Onlooker)\n",
    "            id (int): bee ID\n",
    "            candidate_pos (str): candidate position on the solution surface \\\n",
    "            (the string-encoded architecture in the case of NAS)\n",
    "        '''\n",
    "\n",
    "        if not Logger.EVALUATION_LOGGING:\n",
    "            return\n",
    "\n",
    "        print('\\n{} {} ID ({}) -- Candidate ({})\\n'.format(Logger.__EVAL_PREFIX,\n",
    "                                                           type,\n",
    "                                                           str(id),\n",
    "                                                           str(candidate_pos)))\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def momentum_evaluation_log(candidate, fitness, epochs):\n",
    "        '''Logs momentum evaluation augmentation info\n",
    "\n",
    "        Args:\n",
    "            candidate (str): candidate string representation \\\n",
    "            (architecture in the case of NAS)\n",
    "            fitness (float): fitness value\n",
    "            epochs (int): number of additional momentum epochs assigned\n",
    "        '''\n",
    "\n",
    "        if not Logger.EVALUATION_LOGGING:\n",
    "            return\n",
    "\n",
    "        print('{} Extending ({} - fitness: {}) by {} epochs...\\n'.format(Logger.__MOMENTUM_PREFIX,\n",
    "                                                                         str(candidate),\n",
    "                                                                         fitness,\n",
    "                                                                         epochs))\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def filesave_log(candidate, filename):\n",
    "        '''Logs candidate info upon file-save\n",
    "\n",
    "        Args:\n",
    "            candidate (str): candidate string representation \\\n",
    "            (architecture in the case of NAS)\n",
    "            filename (str): output filename\n",
    "        '''\n",
    "\n",
    "        if not Logger.EVALUATION_LOGGING:\n",
    "            return\n",
    "\n",
    "        print('\\n{} Candidate ({}) was saved to {}\\n'.format(Logger.__FILESAVE_PREFIX,\n",
    "                                                             str(candidate),\n",
    "                                                             filename))\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def start_log():\n",
    "        '''Logs the start msg and intializes the global timer\n",
    "        '''\n",
    "\n",
    "        Logger.__START_TIME = time.time()\n",
    "        dashes = '------------------------'\n",
    "        print('{}\\n-- OPTIMIZATION START --\\n{}'.format(dashes, dashes))\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def end_log():\n",
    "        '''Logs total time taken upon optimization end\n",
    "        '''\n",
    "\n",
    "        end_time = time.time() - Logger.__START_TIME\n",
    "        dashes = '---------------------'\n",
    "        print('{}\\n-- OPTIMIZATION END --\\n{} === TOTAL TIME TAKEN: {} ==== \\n'.format(dashes, dashes, end_time))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Lb79g_x8MmGS"
   },
   "source": [
    "#### Image Augmentations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Ta31nP_TM-nk"
   },
   "outputs": [],
   "source": [
    "\"\"\"Image Augmentation methods\n",
    "\"\"\"\n",
    "\n",
    "class ImgAug:\n",
    "    '''Element-wise image augmentation methods, used to preprocess a\n",
    "    given dataset.\n",
    "\n",
    "    (most affine transformations used are implemented in \\\n",
    "    :class:`tensorflow.keras.preprocessing.image.ImageDataGenerator`)\n",
    "    '''\n",
    "\n",
    "    @staticmethod\n",
    "    def random_cutout(np_tensor, cutout_color=127):\n",
    "        '''Randomly applies cutout augmentation to a given rank 3 tensor as\n",
    "        defined in [1]. Defaults to grey cutout\n",
    "\n",
    "        [1] DeVries, T., & Taylor, G. W. (2017). Improved regularization of\n",
    "        convolutional neural networks with cutout.\n",
    "\n",
    "        Args:\n",
    "            np_tensor (:class:`numpy.array`): rank 3 numpy tensor-respresentation of \\\n",
    "            the data sample\n",
    "            cutout_color (int, optional): RGB-uniform value of the cutout color \\\n",
    "            *(defaults to grey (:code:`127`). white (:code:`255`) and black \\\n",
    "            (:code:`0`) are also valid)*\n",
    "\n",
    "        Returns:\n",
    "            :class:`numpy.array`: augmented numpy tensor (with a random cutout)\n",
    "        '''\n",
    "\n",
    "        cutout_height = int(np.random.uniform(0.1, 0.2) * np_tensor.shape[0])\n",
    "        cutout_width = int(np.random.uniform(0.1, 0.2) * np_tensor.shape[1])\n",
    "\n",
    "        cutout_height_point = np.random.randint(np_tensor.shape[0] - cutout_height)\n",
    "        cutout_width_point = np.random.randint(np_tensor.shape[1] - cutout_width)\n",
    "\n",
    "        np_tensor[cutout_height_point: cutout_height_point + cutout_height,\n",
    "                  cutout_width_point: cutout_width_point + cutout_width,\n",
    "                  :] = cutout_color    # 127 = grey cutout,\n",
    "                                       # 0 (black) or 255 (white) also valid\n",
    "\n",
    "        return np.array(np_tensor)\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def random_contrast(np_tensor):\n",
    "        '''Apply random contrast augmentation\n",
    "\n",
    "        Args:\n",
    "            np_tensor (:class:`numpy.array`): rank 3 numpy tensor-respresentation of \\\n",
    "            the data sample\n",
    "\n",
    "        Returns:\n",
    "            (:class:`numpy.array`): transformed numpy tensor with random contrast\n",
    "        '''\n",
    "\n",
    "        return np.array(random_contrast(np_tensor, 0.5, 2))\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def random_saturation(np_tensor):\n",
    "        '''Apply random saturation augmentation (only works on RGB images, \\\n",
    "        skipped on grayscale datasets)\n",
    "\n",
    "        Args:\n",
    "            np_tensor (:class:`numpy.array`): rank 3 numpy tensor-respresentation of \\\n",
    "            the data sample\n",
    "\n",
    "        Returns:\n",
    "            (:class:`numpy.array`): transformed numpy tensor with random saturation\n",
    "        '''\n",
    "\n",
    "        if np_tensor.shape[-1] != 3:\n",
    "            # not an RGB image, skip augmentation\n",
    "            return np.array(np_tensor)\n",
    "\n",
    "        return np.array(random_saturation(np_tensor, 0.2, 3))\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def augment(np_tensor):\n",
    "        '''Used by ImageDataGenerator's preprocess_function\n",
    "\n",
    "        Args:\n",
    "            np_tensor (:class:`numpy.array`): rank 3 numpy tensor-respresentation of \\\n",
    "            the data sample\n",
    "\n",
    "        Returns:\n",
    "            (:class:`numpy.array`): augmented numpy tensor with all applicable \\\n",
    "            transformations/augmentations\n",
    "        '''\n",
    "\n",
    "        if np.random.uniform() <= Params['CONTRAST_AUG_PROB']:\n",
    "            np_tensor = ImgAug.random_contrast(np_tensor)\n",
    "        if np.random.uniform() <= Params['SATURATION_AUG_PROB']:\n",
    "            np_tensor = ImgAug.random_saturation(np_tensor)\n",
    "        if np.random.uniform() <= Params['CUTOUT_PROB']:\n",
    "            np_tensor = ImgAug.random_cutout(np_tensor)\n",
    "\n",
    "        return np_tensor\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hmD0fOwlw9p4"
   },
   "source": [
    "### Objective Interface"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "FHJWUQkOxGtS"
   },
   "outputs": [],
   "source": [
    "\"\"\"The abstract definition of Objective Interfaces in HiveNAS\n",
    "\"\"\"\n",
    "\n",
    "class ObjectiveInterface(ABC):\n",
    "\n",
    "    '''\n",
    "        Encapsulates the method definitions required to satisfy the hooks\n",
    "        used by the :class:`~core.abc.abc.ArtificialBeeColony` optimizer\n",
    "    '''\n",
    "\n",
    "\n",
    "    @abstractmethod\n",
    "    def sample(self):\n",
    "        '''Samples a random candidate from the optimization surface\n",
    "        *(used primarily by* :class:`~core.abc.scout_bee.ScoutBee` *to initialize the* :class:`~core.abc.food_source.FoodSource`\n",
    "        *vector,* Xm→ *)*\n",
    "\n",
    "        Returns:\n",
    "            str: a string-encoded candidate randomly sampled from the solution space\n",
    "\n",
    "        Raises:\n",
    "            :class:`NotImplementedError`: requires implementation by child class\n",
    "        '''\n",
    "\n",
    "        raise NotImplementedError()\n",
    "\n",
    "\n",
    "    @abstractmethod\n",
    "    def evaluate(self, candidate: str):\n",
    "        '''Evaluates a given string-encoded candidate and returns its fitness score\n",
    "\n",
    "        Args:\n",
    "            candidate (str): string-encoded candidate (an architecture in the case of \\\n",
    "            :class:`~core.nas.nas_interface.NASInterface`)\n",
    "\n",
    "        Returns:\n",
    "            float: the candidate's fitness score\n",
    "\n",
    "        Raises:\n",
    "            :class:`NotImplementedError`: requires implementation by child class\n",
    "        '''\n",
    "\n",
    "        raise NotImplementedError()\n",
    "\n",
    "\n",
    "    @abstractmethod\n",
    "    def get_neighbor(self, candidate: str):\n",
    "        '''Samples a neighbor for a given string-encoded candidate\n",
    "\n",
    "        Args:\n",
    "            candidate (str): the position on the solution surface to find a neighbor for\n",
    "\n",
    "        Returns:\n",
    "            str: the neighboring string-encoded candidate\n",
    "\n",
    "        Raises:\n",
    "            :class:`NotImplementedError`: requires implementation by child class\n",
    "        '''\n",
    "\n",
    "        raise NotImplementedError()\n",
    "\n",
    "\n",
    "    @abstractmethod\n",
    "    def fully_train_best_model(self, from_arch: bool=True):\n",
    "        '''Fully trains the best solution found thus far (exclusively used by NAS)\n",
    "        (relies on paths set in :class:`~config.params.Params`)\n",
    "\n",
    "        Args:\n",
    "            from_arch (bool, optional): determines whether to train model from scratch \\\n",
    "            using the string representations of the architecture (:code:`from_arch = True`) \\\n",
    "            or load the saved model file and continue training (:code:`from_arch = False`). \\\n",
    "            \\\n",
    "            \\\n",
    "            `Note: optimizer settings are typically not saved, \\\n",
    "            therefore training continuation from a model's file can result in a worse overall accuracy` \\\n",
    "            (`read more... <https://stackoverflow.com/a/58693088/3551916>`_).\n",
    "\n",
    "        Returns:\n",
    "            dict: a dictionary containing all relevant results to be saved, including: fitness, \\\n",
    "            number of training epochs conducted (not including any previous trainings), \\\n",
    "            hashed file name, number of trainable parameters\n",
    "\n",
    "        Raises:\n",
    "            :class:`NotImplementedError`: requires implementation by child class\n",
    "        '''\n",
    "\n",
    "        raise NotImplementedError()\n",
    "\n",
    "\n",
    "    @abstractmethod\n",
    "    def momentum_eval(self,\n",
    "                      candidate: str,\n",
    "                      weights_filename: str,\n",
    "                      m_epochs: int):\n",
    "        '''Momentum Evaluation phase (:class:`~core.nas.momentum_eval.MomentumAugmentation`; used exclusively by NAS)\n",
    "\n",
    "        Args:\n",
    "            candidate (str): the selected string-encoded candidate to extend its training\n",
    "            weights_filename (str): the SHA1-hashed unique string ID for the given candidate\n",
    "            m_epochs (int): the additional momentum epochs the candidate should be trained for\n",
    "\n",
    "\n",
    "        Returns:\n",
    "            dict: final fitness value (accuracy) after training continuation\n",
    "\n",
    "        Raises:\n",
    "            :class:`NotImplementedError`: requires implementation by child class\n",
    "        '''\n",
    "\n",
    "        raise NotImplementedError()\n",
    "\n",
    "\n",
    "    @abstractmethod\n",
    "    def is_minimize(self):\n",
    "        '''Used by the optimization algorithm to determine whether this is\n",
    "        a minimization or maximization problem\n",
    "\n",
    "        Returns:\n",
    "            bool: whether to minimize or maximize the fitness (:code:`True` = minimize)\n",
    "\n",
    "        Raises:\n",
    "            :class:`NotImplementedError`: requires implementation by child class\n",
    "        '''\n",
    "\n",
    "        raise NotImplementedError()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "NC7Um7Lb5xxE"
   },
   "source": [
    "### NAS Components"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "c3Isw0Wo2L9_"
   },
   "source": [
    "#### NAS Search Space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "EuFJeLRt1-i_"
   },
   "outputs": [],
   "source": [
    "\"\"\"The Search Space phase of the NAS framework\n",
    "\"\"\"\n",
    "\n",
    "class NASSearchSpace(object):\n",
    "    '''Defines the Search Space used to sample candidates by HiveNAS\n",
    "\n",
    "    Attributes:\n",
    "        all_paths (list): a list of all Directed Acyclic sub-Graphs in the search space (i.e all candidates)\n",
    "        config (dict): the predefined operational parameters pertaining to the search space (defined in :func:`~config.params.Params.search_space_config`)\n",
    "        dag (:class:`~networkx.DiGraph`): the search space graph (depracated; otf-encoding used at the moment)\n",
    "    '''\n",
    "\n",
    "    def __init__(self, config):\n",
    "        '''\n",
    "        Configurations are predefined in the Params class (:func:`~config.params.Params.search_space_config`),\n",
    "        the implementation should work given any set of operations' mapping\n",
    "        and depth\n",
    "\n",
    "        Args:\n",
    "            config (dict): the predefined operational parameters pertaining to the search space (defined in :func:`~config.params.Params.search_space_config`)\n",
    "        '''\n",
    "\n",
    "        self.config = config\n",
    "        # self.__initialize_graph()\n",
    "\n",
    "\n",
    "    def sample(self):\n",
    "        '''\n",
    "        Samples a random point (i.e a candidate architecture) from the search space\n",
    "\n",
    "        Returns:\n",
    "            str: string-encoded representation of the architecture\n",
    "        '''\n",
    "\n",
    "        # assert self.all_paths != None, 'Search space needs to be initialized!'\n",
    "\n",
    "        # idx = np.random.randint(0, len(self.all_paths))\n",
    "        # return self.__encode_path(self.all_paths[idx])\n",
    "\n",
    "        path = ['input']\n",
    "\n",
    "        for l in range(self.config['depth']):\n",
    "\n",
    "            if np.random.rand() < self.config['residual_blocks_rate']:\n",
    "                sc_depth = np.random.randint(1, self.config['depth'] - l + 1)\n",
    "                path.append('L{}_sc_{}'.format(l+1, sc_depth))\n",
    "\n",
    "            path.append('L{}_{}'.format(l+1, np.random.choice(\n",
    "                list(self.config['operations']['search_space'].keys())\n",
    "            )))\n",
    "\n",
    "        path.append('output')\n",
    "\n",
    "        return self.__encode_path(path)\n",
    "\n",
    "\n",
    "    def get_neighbor(self, path_str):\n",
    "        '''Returns a path with 1-op difference (a neighbor).\n",
    "\n",
    "        The definition of a neighbor architecture differs from one model to another in the literature,\n",
    "        however, the general consensus is a 1-op difference network [1].\n",
    "\n",
    "\n",
    "\n",
    "        [1] `Colin White et al. “How Powerful are Performance Predictors in Neural Architecture Search?”\n",
    "        In: Advances in Neural Information Processing Systems 34 (2021).`\n",
    "\n",
    "\n",
    "        Args:\n",
    "            path_str (str): string-encoded representation of the architecture\n",
    "\n",
    "        Returns:\n",
    "            str: string-encoded representation of a neighbor architecture\n",
    "        '''\n",
    "\n",
    "        path = self.__strip_path(self.__decode_path(path_str))\n",
    "\n",
    "        component = np.random.randint(1, len(path) - 1)\n",
    "\n",
    "        ops = []\n",
    "        if path[component].startswith('sc'):\n",
    "            # modify skip-connection (either remove it or change residual depth)\n",
    "            sc_max_depth = len([op for op in path[component:] if not op.startswith('sc')])\n",
    "            ops = [f'sc_{i}' for i in range(sc_max_depth)]\n",
    "            ops.remove(path[component])\n",
    "        else:\n",
    "            # modify operation\n",
    "            ops = list(self.config['operations']['search_space'].keys())\n",
    "            ops.remove(path[component])\n",
    "\n",
    "        # Replace randomly chosen component (operation) with any other op\n",
    "        path[component] = np.random.choice(ops)\n",
    "\n",
    "        # prune skip-connection if op == sc_0\n",
    "        if path[component] == 'sc_0':\n",
    "            del path[component]\n",
    "\n",
    "        return self.__encode_path(path)\n",
    "\n",
    "\n",
    "    def eval_format(self, path):\n",
    "        '''\n",
    "        Formats a path for evaluation (stripped, decoded, and\n",
    "        excluding input/output layers) given a string-encoded path\n",
    "\n",
    "        Args:\n",
    "            path (str): string-encoded representation of the architecture\n",
    "\n",
    "        Returns:\n",
    "            list: a list of operations ([str]) representing a model architecture to be used by the evaluation strategy\n",
    "        '''\n",
    "\n",
    "        return self.__strip_path(self.__decode_path(path))[1:-1]\n",
    "\n",
    "\n",
    "    def __initialize_graph(self):\n",
    "        '''\n",
    "        .. deprecated:: 0.1.0\n",
    "\n",
    "        Initializes the search space DAG for easier sampling by the\n",
    "        search algorithm.\n",
    "\n",
    "        :class:`~networkx.DiGraph` search space encoding consumes too much memory -- deprecated\n",
    "        '''\n",
    "\n",
    "        self.dag = nx.DiGraph()\n",
    "        self.dag.add_node('input')\n",
    "\n",
    "        for l in range(self.config['depth']):\n",
    "            for op in self.config['operations']:\n",
    "                # Connect input layer to first hidden layer\n",
    "                if l == 0:\n",
    "                    self.dag.add_edges_from([('input',\n",
    "                                              'L{}_{}'.format(l+1, op))])\n",
    "                    continue\n",
    "\n",
    "                # Densely connect middle layers\n",
    "                for prev_op in self.config['operations']:\n",
    "                    self.dag.add_edges_from([('L{}_{}'.format(l, prev_op),\n",
    "                                              'L{}_{}'.format(l+1, op))])\n",
    "\n",
    "                # Connect last hidden layer to output stem\n",
    "                if l == self.config['depth'] - 1:\n",
    "                    self.dag.add_edges_from([('L{}_{}'.format(l+1, op),\n",
    "                                              'output')])\n",
    "\n",
    "        self.all_paths = list(nx.all_simple_paths(self.dag, 'input', 'output'))\n",
    "\n",
    "\n",
    "    def __encode_path(self, path):\n",
    "        '''Returns a string encoding of a given path (list of ops)\n",
    "\n",
    "        Args:\n",
    "            path (list): list of operations ([str]) representing the architecture\n",
    "\n",
    "        Returns:\n",
    "            str: string-encoded representation of the given architecture\n",
    "        '''\n",
    "\n",
    "        return '|'.join(self.__strip_path(path))\n",
    "\n",
    "\n",
    "    def __decode_path(self, path):\n",
    "        '''Returns a list of operations given a string-encoded path\n",
    "\n",
    "        Args:\n",
    "            path (str): string-encoded representation of an architecture\n",
    "\n",
    "        Returns:\n",
    "            list: list of operations ([str]) representing the given architecture\n",
    "        '''\n",
    "\n",
    "        ops = path.split('|')\n",
    "\n",
    "        for i in range(1, len(ops) - 1):\n",
    "            ops[i] = 'L{}_{}'.format(i, ops[i])\n",
    "\n",
    "        return ops\n",
    "\n",
    "\n",
    "    def __strip_path(self, path):\n",
    "        '''Strips path of layer ID prefixes given a list of ops\n",
    "\n",
    "        Args:\n",
    "            path (list): list of operations ([str]), each with a layer ID prefix (as was needed for the DAG version of the search space)\n",
    "\n",
    "        Returns:\n",
    "            list: list of operations ([str]) stipped of the layer IDs\n",
    "        '''\n",
    "\n",
    "        return [re.sub('L\\d+_', '', s) for s in path]\n",
    "\n",
    "\n",
    "    def compute_space_size(self):\n",
    "        '''\n",
    "        Returns the number of possible architectures in the given space\n",
    "        (i.e operations and depth) for analytical purposes\n",
    "\n",
    "        Returns:\n",
    "            int: the size of the search space (number of all possible candidates)\n",
    "        '''\n",
    "\n",
    "        return len(list(self.config['operations']['search_space'].keys())) ** \\\n",
    "        self.config['depth']\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "dzZBH0Zu5AUq"
   },
   "source": [
    "#### NAS Evaluation Strategy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "IidYdvn05Djp"
   },
   "outputs": [],
   "source": [
    "\"\"\"The Evaluation Strategy phase of the NAS framework\n",
    "\"\"\"\n",
    "\n",
    "class NASEval(object):\n",
    "    '''Responsible for instantiating and evaluating candidate architectures\n",
    "\n",
    "    Attributes:\n",
    "        config (dict): the predefined operational parameters pertaining to evaluation (defined in :func:`~config.params.Params.evaluation_strategy_config`)\n",
    "        datagen (:class:`~tensorflow.keras.preprocessing.image.ImageDataGenerator`): the data generator used to train all candidates (instantiated once to preserve resources)\n",
    "        model (:class:`~tensorflow.keras.models.Model`): the candidate model to be evaluated (overwritten and deleted with every evaluation process to prevent leaks)\n",
    "    '''\n",
    "\n",
    "    def __init__(self, config):\n",
    "        '''\n",
    "        Initializes the evaluation parameters' configuration ;\n",
    "        for a different dataset, a data-loader must be specified below\n",
    "        as with CIFAR10 Keras loader\n",
    "\n",
    "        Args:\n",
    "            config (dict): the predefined operational parameters pertaining to evaluation (defined in :func:`~config.params.Params.evaluation_strategy_config`)\n",
    "        '''\n",
    "\n",
    "        self.config = config\n",
    "\n",
    "        # specify dataset loaders\n",
    "        if config['dataset'] == 'CIFAR10':\n",
    "            (self.X_train, self.y_train), (self.X_test, self.y_test) = cifar10.load_data()\n",
    "        elif config['dataset'] == 'MNIST':\n",
    "            (self.X_train, self.y_train), (self.X_test, self.y_test) = mnist.load_data()\n",
    "            # Add a placeholder dimension to the dataset to match CIFAR10's\n",
    "            self.X_train = self.X_train.reshape(-1,28,28,1)\n",
    "            self.X_test = self.X_test.reshape(-1,28,28,1)\n",
    "        elif config['dataset'] == 'FASHION_MNIST':\n",
    "            (self.X_train, self.y_train), (self.X_test, self.y_test) = fashion_mnist.load_data()\n",
    "            # Add a placeholder dimension to the dataset to match CIFAR10's\n",
    "            self.X_train = self.X_train.reshape(-1,28,28,1)\n",
    "            self.X_test = self.X_test.reshape(-1,28,28,1)\n",
    "        else:\n",
    "            raise ValueError('Dataset loader undefined!')\n",
    "\n",
    "        if Params['ENABLE_WEIGHT_SAVING']:\n",
    "            # create directory if it does not exist\n",
    "            FileHandler.create_dir(os.path.join(Params.get_results_path(),\n",
    "                                                Params['WEIGHT_FILES_SUBPATH']))\n",
    "\n",
    "        self.__initialize_dataset()\n",
    "\n",
    "\n",
    "    def __instantiate_network(self, arch):\n",
    "        '''Instantiates a Keras network given an architecture op list\n",
    "\n",
    "        Args:\n",
    "            arch (list): a list of architecture operations ([str]), encoded by :class:`~core.nas.search_space.NASSearchSpace`\n",
    "        '''\n",
    "\n",
    "        # residual counters\n",
    "        res_count = []\n",
    "\n",
    "        # add input according to given dataset shape\n",
    "        net = inputs = Input(shape=(self.X_train.shape[1:4]))\n",
    "\n",
    "        # add input stem\n",
    "        for op in self.config['input_stem']:\n",
    "            net = op()(net)\n",
    "\n",
    "        # add hidden layers\n",
    "        for layer in arch:\n",
    "\n",
    "            if layer.startswith('sc'):\n",
    "                # start residual block\n",
    "                res_count.append((net, int(layer[3:])))\n",
    "                continue\n",
    "\n",
    "            assert layer in self.config['operations']['reference_space'], 'Operation must be defined as a partial in HIVE_EVAL_CONFIG'\n",
    "            net = self.config['operations']['reference_space'][layer]()(net)\n",
    "\n",
    "            for idx, row in enumerate(res_count):\n",
    "                connection, counter = row\n",
    "                counter -= 1\n",
    "\n",
    "                # apply pooling to residual blocks to maintain shape\n",
    "                # [deprecated] -- pooling layers padded\n",
    "                # if 'pool' in layer:\n",
    "                #     connection = self.config['operations'][layer]()(connection)\n",
    "\n",
    "                if counter == 0:\n",
    "                    # conv1x1 to normalize channels\n",
    "                    fx = Conv2D(net.shape[-1], (1, 1), padding='same')(connection)\n",
    "                    net = Add()([fx, net])\n",
    "                    del res_count[idx]\n",
    "                else:\n",
    "                    res_count[idx] = (connection, counter)\n",
    "\n",
    "        # add output stem\n",
    "        for op in self.config['output_stem']:\n",
    "            net = op()(net)\n",
    "\n",
    "        # add output layer\n",
    "        net = Dense(len(np.unique(self.y_test)), activation='softmax')(net)\n",
    "\n",
    "        self.model = Model(inputs, net)\n",
    "\n",
    "\n",
    "    def get_weights_filename(self, arch):\n",
    "        '''\n",
    "        Hashes the architecture op-list into a filename using SHA1\n",
    "\n",
    "        Args:\n",
    "            arch (list): a list of architecture operations ([str]), encoded by :class:`~core.nas.search_space.NASSearchSpace`\n",
    "\n",
    "        Returns:\n",
    "            str: SHA1-hashed unique string ID for the given architecture\n",
    "        '''\n",
    "\n",
    "        return hashlib.sha1(''.join(arch).encode(\"UTF-8\")).hexdigest()\n",
    "\n",
    "\n",
    "    def evaluate(self, arch):\n",
    "        '''\n",
    "        Evaluates the candidate architecture given a string-encoded\n",
    "        representation of the model\n",
    "\n",
    "        Args:\n",
    "            arch (list): a list of architecture operations ([str]), encoded by :class:`~core.nas.search_space.NASSearchSpace`\n",
    "\n",
    "        Returns:\n",
    "            dict: a dictionary containing all relevant results to be saved, including: fitness, number of training epochs conducted (in case of ACT), hashed file name, number of trainable parameters, and the last epoch's momentum value if applicable\n",
    "        '''\n",
    "\n",
    "        # instantiate/compile model\n",
    "        self.__instantiate_network(arch)\n",
    "        self.__compile_model()\n",
    "\n",
    "        # train model\n",
    "        # self.model.fit(x=self.X_train,\n",
    "        #         y=self.y_train,\n",
    "        #         batch_size=self.config['batch_size'],\n",
    "        #         epochs=self.config['epochs'],\n",
    "        #         verbose=1,\n",
    "        #         validation_data=(self.X_test, self.y_test))\n",
    "\n",
    "        assert self.config['epochs'] > 2 or self.config['momentum_epochs'] == 0, 'Momentum Augmentation requires at least 3 epochs per candidate'\n",
    "\n",
    "        cb = []\n",
    "        if self.config['termination_threshold_factor'] > 0.0:\n",
    "            cb.append(TerminateOnThreshold(threshold_multiplier=self.config['termination_threshold_factor'],\n",
    "                                           diminishing_factor=self.config['termination_diminishing_factor']))\n",
    "        if self.config['momentum_epochs'] > 0:\n",
    "            cb.append(MomentumAugmentation())\n",
    "\n",
    "        history = self.model.fit(self.datagen.flow(self.X_train,\n",
    "                                                   self.y_train,\n",
    "                                                   shuffle=True,\n",
    "                                                   batch_size=self.config['batch_size'],\n",
    "                                                   subset='training'),\n",
    "                                 validation_data=self.datagen.flow(self.X_train,\n",
    "                                                                   self.y_train,\n",
    "                                                                   batch_size=int(self.config['batch_size'] / 2),\n",
    "                                                                   subset='validation'),\n",
    "                                 epochs=self.config['epochs'],\n",
    "                                 verbose=1,\n",
    "                                 callbacks=cb)\n",
    "\n",
    "        momentum = self.model.momentum if hasattr(self.model, 'momentum') else {}\n",
    "        # test model\n",
    "        eval_res = self.model.evaluate(self.X_test,\n",
    "                                      self.y_test,\n",
    "                                      batch_size=self.config['batch_size'],\n",
    "                                      verbose=1)\n",
    "\n",
    "        # dump training history\n",
    "        hist_path = os.path.join(Params.get_results_path(), Params['HISTORY_FILES_SUBPATH'])\n",
    "        hist_fn = self.get_weights_filename(arch) + '.pickle'\n",
    "\n",
    "        FileHandler.save_pickle(history.history, hist_path, hist_fn)\n",
    "\n",
    "        # save weights for later retraining when needed\n",
    "        filename = self.get_weights_filename(arch) + '.h5'\n",
    "\n",
    "        if Params['ENABLE_WEIGHT_SAVING'] or self.config['momentum_epochs'] > 0:\n",
    "            model_path = os.path.join(Params.get_results_path(), Params['WEIGHT_FILES_SUBPATH'])\n",
    "            self.model.save(model_path + filename)\n",
    "\n",
    "        trainable_params = np.sum([K.count_params(w) for w in self.model.trainable_weights])\n",
    "\n",
    "        # housekeeping\n",
    "        del self.model\n",
    "\n",
    "        # return validation accuracy to maximize + additional data for saving purposes\n",
    "        retval = {\n",
    "            'fitness': eval_res[1],\n",
    "            'epochs': len(history.history['loss']),\n",
    "            'filename': filename,\n",
    "            'params': trainable_params,\n",
    "            'momentum': momentum\n",
    "        }\n",
    "\n",
    "        return retval\n",
    "\n",
    "\n",
    "    def fully_train(self, model_file=None, arch=None):\n",
    "        '''Loads and continues training of a partially-trained model using either a string-encoded architecture (instantiates the model and trains from scratch) or a model h5 file\n",
    "\n",
    "        Args:\n",
    "            model_file (str, optional): a previously saved h5 model file (continues training)\n",
    "            arch (list, optional): a list of architecture operations ([str]), encoded by :class:`~core.nas.search_space.NASSearchSpace`\n",
    "\n",
    "        Returns:\n",
    "            dict: a dictionary containing all relevant results to be saved, including: fitness, number of training epochs conducted (not including any previous trainings), hashed file name, number of trainable parameters\n",
    "        '''\n",
    "\n",
    "        hist_path = os.path.join(Params.get_results_path(), Params['HISTORY_FILES_SUBPATH'])\n",
    "        hist_fn = ''\n",
    "\n",
    "        if model_file is not None:\n",
    "            # load model\n",
    "            self.model = load_model(model_file)\n",
    "            hist_fn =  model_file.split('/')[-1] + '.full.pickle'\n",
    "        else:\n",
    "            # instantiate network\n",
    "            self.__instantiate_network(arch)\n",
    "            self.__compile_model()\n",
    "            model_file = self.get_weights_filename(arch)\n",
    "            hist_fn = model_file + '.full.pickle'\n",
    "\n",
    "        # continue training\n",
    "        # self.model.fit(x=self.X_train,\n",
    "        #                y=self.y_train,\n",
    "        #                batch_size=self.config['batch_size'],\n",
    "        #                epochs=self.config['full_train_epochs'],\n",
    "        #                verbose=1,\n",
    "        #                validation_data=(self.X_test, self.y_test))\n",
    "\n",
    "        history = self.model.fit(self.datagen.flow(self.X_train,\n",
    "                                                   self.y_train,\n",
    "                                                   shuffle=True,\n",
    "                                                   batch_size=self.config['batch_size'],\n",
    "                                                   subset='training'),\n",
    "                                 validation_data=self.datagen.flow(self.X_train,\n",
    "                                                                   self.y_train,\n",
    "                                                                   batch_size=int(self.config['batch_size'] / 2),\n",
    "                                                                   subset='validation'),\n",
    "                                  epochs=self.config['full_train_epochs'],\n",
    "                                  verbose=1)\n",
    "\n",
    "\n",
    "        # test model\n",
    "        eval_res = self.model.evaluate(self.X_test,\n",
    "                                       self.y_test,\n",
    "                                       batch_size=self.config['batch_size'],\n",
    "                                       verbose=1)\n",
    "\n",
    "        # dump training history\n",
    "        FileHandler.save_pickle(history.history, hist_path, hist_fn)\n",
    "\n",
    "        # save weights for later retraining when needed\n",
    "        # if Params['ENABLE_WEIGHT_SAVING']:\n",
    "        # Save fully trained model regardless of params\n",
    "        self.model.save(model_file + '.full.h5')\n",
    "\n",
    "        trainable_params = np.sum([K.count_params(w) for w in self.model.trainable_weights])\n",
    "\n",
    "        # housekeeping\n",
    "        del self.model\n",
    "\n",
    "        retval = {\n",
    "            'fitness': eval_res[1],\n",
    "            'epochs': len(history.history['loss']),\n",
    "            'filename': model_file,\n",
    "            'params': trainable_params\n",
    "        }\n",
    "\n",
    "        return retval\n",
    "\n",
    "\n",
    "    def momentum_training(self, weights_file, m_epochs):\n",
    "        '''Loads and continues training of a partially-trained model\n",
    "\n",
    "        Args:\n",
    "            weights_file (str): the previously saved h5 model file (continues training)\n",
    "            m_epochs (int): number of momentum epochs to continue training for\n",
    "\n",
    "        Returns:\n",
    "            dict: final fitness value (accuracy) after training continuation\n",
    "        '''\n",
    "\n",
    "        # load model\n",
    "        self.model = load_model(weights_file)\n",
    "        # hist_fn = model_file.split('/')[-1] + '.ma.pickle'\n",
    "\n",
    "        history = self.model.fit(self.datagen.flow(self.X_train,\n",
    "                                                   self.y_train,\n",
    "                                                   shuffle=True,\n",
    "                                                   batch_size=self.config['batch_size'],\n",
    "                                                   subset='training'),\n",
    "                                 validation_data=self.datagen.flow(self.X_train,\n",
    "                                                                   self.y_train,\n",
    "                                                                   batch_size=int(self.config['batch_size'] / 2),\n",
    "                                                                   subset='validation'),\n",
    "                                  epochs=m_epochs,\n",
    "                                  verbose=1)\n",
    "\n",
    "\n",
    "        # test model\n",
    "        eval_res = self.model.evaluate(self.X_test,\n",
    "                                       self.y_test,\n",
    "                                       batch_size=self.config['batch_size'],\n",
    "                                       verbose=1)\n",
    "\n",
    "        # dump training history\n",
    "        # FileHandler.save_pickle(history.history, hist_path, hist_fn)\n",
    "\n",
    "        # Save continued-training model\n",
    "        self.model.save(weights_file)\n",
    "\n",
    "        # housekeeping\n",
    "        del self.model\n",
    "        gc.collect()\n",
    "\n",
    "        retval = {\n",
    "            'fitness': eval_res[1]\n",
    "        }\n",
    "\n",
    "        return retval\n",
    "\n",
    "\n",
    "    def __initialize_dataset(self):\n",
    "        '''\n",
    "        Prepares the dataset\n",
    "\n",
    "        Preprocessing includes standardization and affine transformation (if applicable)\n",
    "\n",
    "        Initializes the :class:`tensorflow.keras.preprocessing.image.ImageDataGenerator` for evaluation\n",
    "        '''\n",
    "\n",
    "        # Standardize data\n",
    "        X_train_mean = np.mean(self.X_train, axis=(0,1,2))\n",
    "        X_train_std = np.std(self.X_train, axis=(0,1,2))\n",
    "        self.X_train = (self.X_train - X_train_mean) / X_train_std\n",
    "        self.X_test = (self.X_test - X_train_mean) / X_train_std\n",
    "\n",
    "        # Affine transformations\n",
    "        if Params['AFFINE_TRANSFORMATIONS_ENABLED']:\n",
    "            self.datagen = ImageDataGenerator(\n",
    "                zoom_range = [0.8, 1.1],\n",
    "                shear_range= 10,\n",
    "                rotation_range=15,\n",
    "                width_shift_range=0.1,\n",
    "                height_shift_range=0.1,\n",
    "                horizontal_flip=True,\n",
    "                preprocessing_function=ImgAug.augment,\n",
    "                validation_split=0.2\n",
    "            )\n",
    "        else:\n",
    "            self.datagen = ImageDataGenerator(preprocessing_function=ImgAug.augment,\n",
    "                                              validation_split=0.2)\n",
    "\n",
    "        # per docs, ImageDataGenerator.fit() is only needed if the generator enables:\n",
    "        # featurewise_center or featurewise_std_normalization or zca_whitening\n",
    "        # self.datagen.fit(self.X_train)\n",
    "\n",
    "        # # One-hot encoding\n",
    "        # deprecated; memory consumption too high for intermediate tensors\n",
    "        # self.y_train = utils.to_categorical(self.y_train)\n",
    "        # self.y_test = utils.to_categorical(self.y_test)\n",
    "\n",
    "\n",
    "    def __compile_model(self):\n",
    "        '''Compiles model in preparation for evaluation\n",
    "        '''\n",
    "\n",
    "        self.model.compile(loss='sparse_categorical_crossentropy', \\\n",
    "                           optimizer=self.config['optimizer'](), \\\n",
    "                           metrics=['sparse_categorical_accuracy'])\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "pAoHETeqfoDv"
   },
   "source": [
    "##### Adaptive Cutoff Threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ziIY3qNmfs-p"
   },
   "outputs": [],
   "source": [
    "\"\"\"Calculates a cutoff performance threshold, below which a model stops training\n",
    "\"\"\"\n",
    "\n",
    "class TerminateOnThreshold(Callback):\n",
    "    '''Adaptive Cutoff Threshold (ACT)\n",
    "\n",
    "    Keras Callback that terminates training if a given :code:`val_sparse_categorical_accuracy`\n",
    "    dynamic threshold is not reached after ε epochs.\n",
    "    The termination threshold has a logarithmic nature where the threshold\n",
    "    increases by a decaying factor.\n",
    "\n",
    "    Attributes:\n",
    "        beta (float): threshold coefficient (captures the leniency of the calculated threshold)\n",
    "        monitor (str): the optimizer metric type to monitor and calculate ACT on\n",
    "        n_classes (int): number of classes\n",
    "        zeta (float): diminishing factor; a positive, non-zero factor that controls how steeply the function horizontally asymptotes at y = 1.0 (i.e 100% accuracy)\n",
    "    '''\n",
    "\n",
    "    def __init__(self,\n",
    "                monitor='val_sparse_categorical_accuracy',\n",
    "                threshold_multiplier=0.25,\n",
    "                diminishing_factor=0.25,\n",
    "                n_classes = None):\n",
    "        '''Initialize threshold-based termination callback\n",
    "\n",
    "        Args:\n",
    "            monitor (str, optional): the optimizer metric type to monitor and calculate ACT on\n",
    "            threshold_multiplier (float, optional): threshold coefficient (captures the leniency of the calculated threshold)\n",
    "            diminishing_factor (float, optional): iminishing factor; a positive, non-zero factor that controls how steeply the function horizontally asymptotes at y = 1.0 (i.e 100% accuracy)\n",
    "            n_classes (None, optional): number of classes / output neurons\n",
    "        '''\n",
    "\n",
    "        super(TerminateOnThreshold, self).__init__()\n",
    "\n",
    "        self.monitor = monitor\n",
    "        self.beta = threshold_multiplier\n",
    "        self.zeta = diminishing_factor\n",
    "        self.n_classes = n_classes\n",
    "\n",
    "    def get_threshold(self, epoch):\n",
    "        '''Calculates the termination threshold given the current epoch\n",
    "\n",
    "            ΔThreshold = ß(1 - (1 / n))\n",
    "            Threshold_base = (1 / n) + ΔThreshold = (1 / n) + ß(1 - (1 / n))\n",
    "                                                  = (1 + ßn - ß) / n\n",
    "            Range of Threshold_base = (1 / n, 1) ; horizontal asymptote at 1\n",
    "            ΔThreshold decays as the number of classes decreases\n",
    "\n",
    "            --------------\n",
    "\n",
    "            To account for the expected increase in accuracy over the number\n",
    "            of epochs ε, a growth_factor is added to the base threshold:\n",
    "            growth_factor = (1 - Threshold_base) - (1 / (1 / 1-Threshold_base) + ζ(ε - 1))\n",
    "\n",
    "            Threshold_adaptive = Threshold_base + growth_factor\n",
    "            Range of growth_factor = [Threshold_base, 1) ; horizontal asymptote at 1\n",
    "\n",
    "        Args:\n",
    "            epoch (int): current epoch\n",
    "\n",
    "        Returns:\n",
    "            float: calculated cutoff threshold\n",
    "        '''\n",
    "\n",
    "        baseline = 1.0 / self.n_classes     # baseline (random) val_acc\n",
    "        complement_baseline = 1 - baseline\n",
    "        delta_threshold = complement_baseline * self.beta\n",
    "        base_threshold = baseline + delta_threshold\n",
    "        ''' n_classes = 10, threshold_multiplier = 0.15 '''\n",
    "        ''' yields .325 acc threshold for epoch 1 '''\n",
    "\n",
    "        # epoch-based decaying increase in val_acc threshold\n",
    "        complement_threshold = 1 - base_threshold    # the increase factor's upper limit\n",
    "        growth_denom = (1.0 / complement_threshold) + self.zeta * (epoch - 1)\n",
    "        growth_factor = complement_threshold - 1.0 / growth_denom\n",
    "\n",
    "        calculated_threshold = base_threshold + growth_factor\n",
    "        '''\n",
    "            Same settings as before yields:\n",
    "            epoch 1 = .325000\n",
    "            epoch 2 = .422459,\n",
    "            epoch 3 = .495327,\n",
    "            epoch 4 = .551867,\n",
    "            epoch 5 = .597014\n",
    "        '''\n",
    "\n",
    "        return calculated_threshold\n",
    "\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs=None):\n",
    "        '''Called by Keras backend after each epoch during :code:`.fit()` & :code:`.evaluate()`\n",
    "\n",
    "        Args:\n",
    "            epoch (int): current epoch\n",
    "            logs (None, optional): contains all the monitors (or metrics) used by the optimizer in the training and evaluation contexts\n",
    "        '''\n",
    "\n",
    "        logs = logs or {}\n",
    "\n",
    "        if self.model is None:\n",
    "            return\n",
    "\n",
    "        if self.n_classes is None:\n",
    "            self.n_classes = self.model.layers[-1].output_shape[1]\n",
    "\n",
    "        threshold = self.get_threshold(epoch + 1)\n",
    "\n",
    "        if self.monitor in logs:\n",
    "            val_acc = logs[self.monitor]\n",
    "            if val_acc < threshold:\n",
    "                # threshold not met, terminate\n",
    "                print(f'\\nEpoch {(epoch + 1)}: Accuracy ({val_acc}) has not reached the baseline threshold {threshold}, terminating training... \\n')\n",
    "                self.model.stop_training = True\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4MabBEyDNwZB"
   },
   "outputs": [],
   "source": [
    "''' TESTING THRESHOLD EVAL '''\n",
    "threshold_multiplier = 0.25\n",
    "diminishing_factor = 0.25\n",
    "n_classes = 10\n",
    "n_epochs = 50\n",
    "\n",
    "thresholds = []\n",
    "for epoch in range(1, n_epochs + 1):\n",
    "\n",
    "    baseline = 1.0 / n_classes     # baseline (random) val_acc\n",
    "    complement_baseline = 1 - baseline\n",
    "    delta_threshold = complement_baseline * threshold_multiplier\n",
    "    base_threshold = baseline + delta_threshold\n",
    "\n",
    "    # epoch-based decaying increase in val_acc threshold\n",
    "    complement_threshold = 1 - base_threshold    # the increase factor's upper limit\n",
    "    growth_denom = (1.0 / complement_threshold) + diminishing_factor * (epoch - 1)\n",
    "    growth_factor = complement_threshold - 1.0 / growth_denom\n",
    "\n",
    "    calculated_threshold = base_threshold + growth_factor\n",
    "\n",
    "    thresholds.append(calculated_threshold)\n",
    "\n",
    "plt.plot(thresholds, '.')\n",
    "plt.show()\n",
    "\n",
    "# print first 5 thresholds\n",
    "print('\\n\\n')\n",
    "[print(f'Epoch {t+1} threshold: {thresholds[t]}') for t in range(0, 5)];\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cj1xmJPigB_2"
   },
   "source": [
    "##### Momentum Augmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "GSHOs7sHgHTq"
   },
   "outputs": [],
   "source": [
    "\"\"\"Calculates and incentivizes the stability of convergence\n",
    "\"\"\"\n",
    "\n",
    "class MomentumAugmentation(Callback):\n",
    "    '''Calculates the momentum's moving average of the parent model\n",
    "\n",
    "    Attributes:\n",
    "        monitor (str): the optimizer metric type to monitor and calculate momentums on\n",
    "    '''\n",
    "\n",
    "    def __init__(self, monitor='val_sparse_categorical_accuracy'):\n",
    "        '''Initialize MA\n",
    "\n",
    "        Args:\n",
    "            monitor (str, optional): the optimizer metric type to monitor and calculate momentums on\n",
    "        '''\n",
    "\n",
    "        super(MomentumAugmentation, self).__init__()\n",
    "        self.monitor = monitor\n",
    "\n",
    "\n",
    "    def get_momentum(self, epoch, acc):\n",
    "        '''Calculates the momentums based on the given accuracies and epochs\n",
    "\n",
    "        μm(ε) = (αm(ε) − αm(ε − 1)) / (αm(ε − 1) − αm(ε − 2))  for all ε ≥ 2\n",
    "\n",
    "        Args:\n",
    "            epoch (int): current epoch\n",
    "            acc (float): current epoch's accuracy\n",
    "\n",
    "        Returns:\n",
    "            (float, float): a tuple consisting of the (current accuracy, current momentum)\n",
    "        '''\n",
    "\n",
    "        if epoch < 2:\n",
    "            # momentum = acc at ε < 3\n",
    "            return (acc, acc)\n",
    "\n",
    "        delta_1 = acc - self.model.momentum[epoch - 1][0]\n",
    "        delta_2 = self.model.momentum[epoch - 1][0] - self.model.momentum[epoch - 2][0]\n",
    "\n",
    "        if delta_2 == 0.0:\n",
    "            # avoid division by 0\n",
    "            # if previous 2 accuracies are somehow exactly the same (very unlikely) => 0 momentum\n",
    "            return (acc, 0.0)\n",
    "\n",
    "        current_momentum = delta_1 / delta_2\n",
    "\n",
    "        return (acc, current_momentum)\n",
    "\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs=None):\n",
    "        '''Called by Keras backend after each epoch during :code:`.fit()` & :code:`.evaluate()`\n",
    "\n",
    "        Args:\n",
    "            epoch (int): current epoch\n",
    "            logs (dict, optional): contains all the monitors (or metrics) used by the optimizer in the training and evaluation contexts\n",
    "        '''\n",
    "\n",
    "        logs = logs or {}\n",
    "\n",
    "        if self.model is None:\n",
    "            return\n",
    "\n",
    "        if not hasattr(self.model, 'momentum'):\n",
    "            self.model.momentum = {}\n",
    "\n",
    "        if self.monitor in logs:\n",
    "            val_acc = logs[self.monitor]\n",
    "\n",
    "            self.model.momentum[epoch] = self.get_momentum(epoch, val_acc)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "FQl0Y7o5iBM8"
   },
   "source": [
    "#### NAS Interface"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "kkz4zLd92Ume"
   },
   "outputs": [],
   "source": [
    "\"\"\"The NAS Interface encapsulating the Evaluation Strategy and Search Space\n",
    "\"\"\"\n",
    "\n",
    "class NASInterface(ObjectiveInterface):\n",
    "    '''\n",
    "    An interface that combines the Search Space & Evaluation Strategy\n",
    "    for the NAS Search Algorithm (ABC)\n",
    "\n",
    "    Attributes:\n",
    "        cls.eval_strategy (:class:`~core.nas.evaluation_strategy.NASEval`): NASEval instance used to instantiate and evaluate candidates\n",
    "        cls.search_space (:class:`~core.nas.search_space.NASSearchSpace`): NASSearchSpace instance used to sample candidates and neighbors\n",
    "    '''\n",
    "\n",
    "    def __init__(self,\n",
    "                 space_config=None,\n",
    "                 eval_config=None):\n",
    "        '''Initializes the search space and evaluator\n",
    "\n",
    "        Args:\n",
    "            space_config (dict, optional): the predefined operational parameters pertaining to the search space (defined in :func:`~config.params.Params.search_space_config`)\n",
    "            eval_config (dict, optional): the predefined operational parameters pertaining to evaluation (defined in :func:`~config.params.Params.evaluation_strategy_config`)\n",
    "        '''\n",
    "\n",
    "        space_config = space_config or Params.search_space_config()\n",
    "        eval_config = eval_config or Params.evaluation_strategy_config()\n",
    "\n",
    "        NASInterface.search_space = NASSearchSpace(space_config)\n",
    "        NASInterface.eval_strategy = NASEval(eval_config)\n",
    "\n",
    "\n",
    "    def sample(self):\n",
    "        '''Samples new random candidate architecture from the search space\n",
    "\n",
    "        Returns:\n",
    "            str: string-encoded representation of the sampled candidate architecture\n",
    "        '''\n",
    "\n",
    "        return NASInterface.search_space.sample()\n",
    "\n",
    "\n",
    "    def evaluate(self, candidate):\n",
    "        '''Evaluates a given candidate architecture\n",
    "\n",
    "        Args:\n",
    "            candidate (str): string-encoded representation of the architecture to be evaluated\n",
    "\n",
    "        Returns:\n",
    "            dict: a dictionary containing all relevant results to be saved, including: fitness, number of training epochs conducted (in case of ACT), hashed file name, number of trainable parameters, and the last epoch's momentum value if applicable\n",
    "        '''\n",
    "\n",
    "        formatted = NASInterface.search_space.eval_format(candidate)\n",
    "        res = NASInterface.eval_strategy.evaluate(formatted)\n",
    "\n",
    "        # housekeeping\n",
    "        K.clear_session()\n",
    "        gc.collect()\n",
    "\n",
    "        return res\n",
    "\n",
    "\n",
    "    def get_neighbor(self, orig_arch):\n",
    "        '''Returns a random architecture with 1 op diff to the given candidate\n",
    "\n",
    "        Args:\n",
    "            orig_arch (str): string-encoded representation of the candidate architecture\n",
    "\n",
    "        Returns:\n",
    "            str: string-encoded representation of the neighbor architecture\n",
    "        '''\n",
    "\n",
    "        return NASInterface.search_space.get_neighbor(orig_arch)\n",
    "\n",
    "\n",
    "    def fully_train_best_model(self, from_arch=True):\n",
    "        '''\n",
    "        Fully-train best-performing model\n",
    "        (relies on paths set in :class:`~config.params.Params`)\n",
    "\n",
    "        Args:\n",
    "            from_arch (bool, optional): determines whether to train model from scratch \\\n",
    "            using the string representations of the architecture (:code:`from_arch = True`) \\\n",
    "            or load the saved model file and continue training (:code:`from_arch = False`). \\\n",
    "            \\\n",
    "            `Note: optimizer settings are typically not saved, \\\n",
    "            therefore training continuation from a model's file can result in a worse overall accuracy` \\\n",
    "            (`read more... <https://stackoverflow.com/a/58693088/3551916>`_).\n",
    "\n",
    "        Returns:\n",
    "            dict: a dictionary containing all relevant results to be saved, including: fitness, number of training epochs conducted (not including any previous trainings), hashed file name, number of trainable parameters\n",
    "\n",
    "        Raises:\n",
    "            :class:`FileNotFoundError`: raises an error if the results file or model h5 file (when applicable) do not exist\n",
    "        '''\n",
    "\n",
    "        # check existence of results file\n",
    "        filename = f'{Params[\"CONFIG_VERSION\"]}.csv'\n",
    "        results_file = os.path.join(Params.get_results_path(), filename)\n",
    "        FileHandler.path_must_exist(results_file)    # breaks if file does not exist\n",
    "\n",
    "        # extract best fitness weight file\n",
    "        results_df = pd.read_csv(results_file, header=0, index_col=0)\n",
    "        weight_file = results_df.loc[results_df['fitness'] == results_df['fitness'].max(), 'weights_filename'].values[0]\n",
    "        arch = results_df.loc[results_df['fitness'] == results_df['fitness'].max(), 'candidate'].values[0]\n",
    "\n",
    "        print(f'\\nFound best-performing model {{{arch}}} with a fitness score of {results_df[\"fitness\"].max()}\\n')\n",
    "\n",
    "        # housekeeping -> results_df no longer needed and is potentially large\n",
    "        del results_df\n",
    "        gc.collect()\n",
    "\n",
    "        if from_arch:\n",
    "            # Retrains from scratch given the network arch\n",
    "            arch = formatted = NASInterface.search_space.eval_format(arch)\n",
    "            return NASInterface.eval_strategy.fully_train(arch=arch)\n",
    "\n",
    "        # check existence of weight file\n",
    "        weight_file = os.path.join(Params.get_results_path(), Params['WEIGHT_FILES_SUBPATH'], weight_file)\n",
    "        FileHandler.path_must_exist(weight_file)    # breaks if file does not exist\n",
    "\n",
    "\n",
    "        # Continues training from saved h5 model (often results in lower fitness)\n",
    "        return NASInterface.eval_strategy.fully_train(model_file=weight_file)\n",
    "\n",
    "\n",
    "    def momentum_eval(self, candidate, weights_filename, m_epochs):\n",
    "        '''Trains a given network for additional :code:`m_epochs`\n",
    "\n",
    "        Args:\n",
    "            candidate (str): string-encoded representation of the candidate architecture\n",
    "            weights_filename (str): the SHA1-hashed unique string ID for the given architecture\n",
    "            m_epochs (int): the additional momentum epochs the candidate should be trained for\n",
    "\n",
    "        Returns:\n",
    "            dict: final fitness value (accuracy) after training continuation\n",
    "        '''\n",
    "\n",
    "        # check existence of weight file\n",
    "        weights_path = os.path.join(Params.get_results_path(), Params['WEIGHT_FILES_SUBPATH'], weights_filename)\n",
    "        FileHandler.path_must_exist(weights_path)    # breaks if file does not exist\n",
    "\n",
    "\n",
    "        # Continues training from saved h5 model (often results in lower fitness)\n",
    "        return NASInterface.eval_strategy.momentum_training(weights_path, m_epochs)\n",
    "\n",
    "\n",
    "    @property\n",
    "    def is_minimize(self):\n",
    "        '''\n",
    "        Used by the optimization algorithm to determine whether this is\n",
    "        a minimization or maximization problem\n",
    "\n",
    "        Returns:\n",
    "            bool: hard-coded :code:`False`; the search algorithm is always maximizing accuracy\n",
    "        '''\n",
    "\n",
    "        return False\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ntNIB5vO8T5a"
   },
   "source": [
    "### Numerical Optimization Benchmarks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Q5Hg6jF_g_d9"
   },
   "source": [
    "#### NumericalBenchmark Base Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "KccufdV08X3b"
   },
   "outputs": [],
   "source": [
    "\"\"\"Abstract :class:`~core.objective_interface.ObjectiveInterface` class for \\\n",
    "numerical benchmarks\n",
    "\"\"\"\n",
    "\n",
    "class NumericalBenchmark(ObjectiveInterface):\n",
    "    '''Abstract class for Numerical Optimization Benchmarks\n",
    "\n",
    "    Attributes:\n",
    "        dim (int): number of dimensions for the given benchmark\n",
    "        maxv (float): maximum value in the search space\n",
    "        minv (float): minimum value in the search space\n",
    "        minimization (bool): determines whether this is a minimization \\\n",
    "        or maximization problem\n",
    "    '''\n",
    "\n",
    "    def __init__(self, dim, minv, maxv, minimization):\n",
    "        '''Initialize the numerical benchmark\n",
    "\n",
    "        Args:\n",
    "            dim (int): number of dimensions for the given benchmark\n",
    "            maxv (float): maximum value in the search space\n",
    "            minv (float): minimum value in the search space\n",
    "            minimization (bool): determines whether this is a minimization \\\n",
    "            or maximization problem\n",
    "        '''\n",
    "\n",
    "        self.dim = dim\n",
    "        self.minv = minv\n",
    "        self.maxv = maxv\n",
    "        self.minimization = minimization\n",
    "\n",
    "\n",
    "    def sample(self):\n",
    "        '''Samples a random point from the objective function\n",
    "\n",
    "        Returns:\n",
    "            float: randomly sample a candidate from the given space\n",
    "        '''\n",
    "\n",
    "        return np.random.uniform(low=self.minv, high=self.maxv, \\\n",
    "                                 size=self.dim)\n",
    "\n",
    "\n",
    "    def get_neighbor(self, pos):\n",
    "        '''Finds a random neighbor by displacing 1 positional component by :math:`\\\\phi`\n",
    "\n",
    "        Args:\n",
    "            pos (list): list of positional components (floats)\n",
    "\n",
    "        Returns:\n",
    "            list: a neighboring position\n",
    "        '''\n",
    "\n",
    "        op = np.random.choice(pos)\n",
    "        phi = np.random.uniform(low=-1, high=1, size=len(pos))\n",
    "        neighbor_pos = pos + (pos - op) * phi\n",
    "\n",
    "        return self.__eval_boundary(neighbor_pos)\n",
    "\n",
    "\n",
    "    def __eval_boundary(self, n_pos):\n",
    "        '''Ensures the newly sampled position (neighbor) lies within the\n",
    "        search space boundaries; if it is beyond the boundary, it snaps \\\n",
    "        to the edge\n",
    "\n",
    "        Args:\n",
    "            n_pos (list): the new position to be evaluated (list of floats)\n",
    "\n",
    "        Returns:\n",
    "            bool: whether or not the new position is within boundaries\n",
    "        '''\n",
    "\n",
    "        if (n_pos < self.minv).any() or \\\n",
    "        (n_pos > self.maxv).any():\n",
    "            n_pos[n_pos < self.minv] = self.minv\n",
    "            n_pos[n_pos > self.maxv] = self.maxv\n",
    "\n",
    "        return n_pos\n",
    "\n",
    "\n",
    "    @property\n",
    "    def minimum(self):\n",
    "        \"\"\"The minimum value getter\n",
    "\n",
    "        Returns:\n",
    "            float: minimum value for a position component\n",
    "        \"\"\"\n",
    "\n",
    "        return self.minv\n",
    "\n",
    "\n",
    "    @property\n",
    "    def maximum(self):\n",
    "        \"\"\"The maximum value getter\n",
    "\n",
    "        Returns:\n",
    "            float: maximum value for a position component\n",
    "        \"\"\"\n",
    "\n",
    "        return self.maxv\n",
    "\n",
    "\n",
    "    @property\n",
    "    def is_minimize(self):\n",
    "        \"\"\"Minimization toggle getter\n",
    "\n",
    "        Returns:\n",
    "            bool: determines whether this is a minimization or \\\n",
    "            maximization problem\n",
    "        \"\"\"\n",
    "\n",
    "        return self.minimization\n",
    "\n",
    "\n",
    "    def fully_train_best_model(self, from_arch: bool=True):\n",
    "        '''Exception handler for :code:`fully_train_best_model` on NumericalBenchmarks\n",
    "\n",
    "        Raises:\n",
    "            :class:`ValueError`: raised when :code:`fully_train_best_model` is called on a NumericalBenchmark\n",
    "\n",
    "        Args:\n",
    "            from_arch (bool, optional): N/A\n",
    "        '''\n",
    "\n",
    "        raise ValueError('\\'ObjectiveInterface.fully_train_best_model()\\' \\\n",
    "        called for a Numerical Benchmark')\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "IGYJZBG-gH6s"
   },
   "source": [
    "#### Sphere"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "70x3Oco4-csV"
   },
   "outputs": [],
   "source": [
    "\"\"\"Shere optimization benchmark\n",
    "\"\"\"\n",
    "\n",
    "class Sphere(NumericalBenchmark):\n",
    "    '''Sphere optimization benchmark as given by [ Σ Xi^2 , for i=1 in dim ]\n",
    "    '''\n",
    "\n",
    "    def __init__(self, dim, is_minimization=True):\n",
    "        '''Initialize the benchmark\n",
    "\n",
    "        Args:\n",
    "            dim (int): number of dimensions / position components\n",
    "            is_minimization (bool, optional): determines whether to minimize \\\n",
    "            or maximize the sphere; defaults to minimization\n",
    "        '''\n",
    "\n",
    "        super(Sphere, self).__init__(dim, -100.0, 100.0, is_minimization)\n",
    "\n",
    "\n",
    "    def evaluate(self, pos):\n",
    "        '''Evaluate a given position\n",
    "\n",
    "        Args:\n",
    "            pos (list): list of position components (floats)\n",
    "\n",
    "        Returns:\n",
    "            dict: fitness value of the given position along with the static data required by \\\n",
    "            :class:`~core.objective_interface.ObjectiveInterface`\n",
    "        '''\n",
    "\n",
    "        return {\n",
    "            'fitness': sum(np.power(pos, 2)),\n",
    "            'epochs': 1,\n",
    "            'filename': '',\n",
    "            'params': self.dim,\n",
    "            'momentum': {'': ('', '')}\n",
    "        }\n",
    "\n",
    "\n",
    "    def momentum_eval(self, pos, weights, m_epochs):\n",
    "        '''Redundant implementation as :func:`~benchmarks.sphere.Sphere.evaluate` \\\n",
    "        to satisfy the :class:`~core.objective_interface.ObjectiveInterface` hooks\n",
    "\n",
    "        Args:\n",
    "            pos (list): list of position components (floats)\n",
    "            weights (str): N/A\n",
    "            m_epochs (int): N/A\n",
    "\n",
    "        Returns:\n",
    "            dict: a dictionary containing the evaluated fitness value\n",
    "        '''\n",
    "\n",
    "        return {\n",
    "            'fitness': sum(np.power(pos, 2))\n",
    "        }\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kcAjG8NJgKry"
   },
   "source": [
    "#### Rosenbrock"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "dm2x7F2V_VsV"
   },
   "outputs": [],
   "source": [
    "\"\"\"Rosenbrock optimization benchmark\n",
    "\"\"\"\n",
    "\n",
    "class Rosenbrock(NumericalBenchmark):\n",
    "    ''' The benchmark uses the :class:`scipy.optimize.rosen` version of \\\n",
    "    Rosenbrock, as given by\n",
    "\n",
    "    [ Σ { 100(Xi+1 - Xi)^2 + (Xi - 1)^2 } , for i=1 in dim - 1 ]\n",
    "\n",
    "    '''\n",
    "\n",
    "    def __init__(self, dim):\n",
    "        '''Initializes the benchmark\n",
    "\n",
    "        Args:\n",
    "            dim (int): number of dimensions that constitute the position\n",
    "        '''\n",
    "\n",
    "        super(Rosenbrock, self).__init__(dim, -30.0, 30.0, True)\n",
    "\n",
    "\n",
    "    def evaluate(self, pos):\n",
    "        '''Evaluate the given position\n",
    "\n",
    "        Args:\n",
    "            pos (list): list of floats representing the position components\n",
    "\n",
    "        Returns:\n",
    "            dict: fitness value of the given position along with the static data required by \\\n",
    "            :class:`~core.objective_interface.ObjectiveInterface`\n",
    "        '''\n",
    "\n",
    "        return {\n",
    "            'fitness': optimize.rosen(pos),\n",
    "            'epochs': 1,\n",
    "            'filename': '',\n",
    "            'params': self.dim,\n",
    "            'momentum': {'': ('', 0)}\n",
    "        }\n",
    "\n",
    "\n",
    "    def momentum_eval(self, pos, weights, m_epochs):\n",
    "        '''Redundant implementation as :func:`~benchmarks.rosenbrock.Rosenbrock.evaluate` \\\n",
    "        to satisfy the :class:`~core.objective_interface.ObjectiveInterface` hooks\n",
    "\n",
    "        Args:\n",
    "            pos (list): list of position components (floats)\n",
    "            weights (str): N/A\n",
    "            m_epochs (int): N/A\n",
    "\n",
    "        Returns:\n",
    "            dict: a dictionary containing the evaluated fitness value\n",
    "        '''\n",
    "\n",
    "        return {\n",
    "            'fitness': optimize.rosen(pos)\n",
    "        }\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vDcYRCilw2bZ"
   },
   "source": [
    "### Artificial Bee Colony Components"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wYG3yKSEgOC8"
   },
   "source": [
    "#### FoodSource"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "XsJNUMOiVRwx"
   },
   "outputs": [],
   "source": [
    "\"\"\"The FoodSources representing positions on the optimization surface\n",
    "used by the ABC optimizer as desribed in [1]. Can be considered the\n",
    "*memory units* for Employee and Onlooker Bees\n",
    "\n",
    "    [1] Karaboga, D., & Basturk, B. (2007). A powerful and efficient\n",
    "    algorithm for numerical function optimization: artificial bee\n",
    "    colony (ABC) algorithm. Journal of global optimization, 39(3),\n",
    "    459-471.\n",
    "\"\"\"\n",
    "\n",
    "class FoodSource(object):\n",
    "    '''The FoodSource class encapsulates the position on the optimization surface and its\n",
    "    corresponding fitness value + the evaluation time taken (in seconds)\n",
    "\n",
    "    Attributes:\n",
    "        eval_time (float): the time taken to evaluate the given position (in seconds)\n",
    "        pos (str): the string-encoded position on the optimization surface\n",
    "        fit (float): the fitness corresponding the stored position\n",
    "    '''\n",
    "\n",
    "    def __init__(self, position=None, fitness=None):\n",
    "        '''\n",
    "        Data structure containing a FoodSource (position on the\n",
    "        optimization surface) and its fitness value\n",
    "\n",
    "        Args:\n",
    "            position (str, optional): the string-encoded position on the optimization surface\n",
    "            fitness (float, optional): the fitness corresponding the stored position\n",
    "        '''\n",
    "\n",
    "        self.pos = position\n",
    "        self.fit = fitness\n",
    "        self.eval_time = 0.0\n",
    "\n",
    "\n",
    "    def encode_position(self):\n",
    "        '''Returns an encoded position for use in dicts\n",
    "\n",
    "        Returns:\n",
    "            str: a formatted, whitespace-stripped version of the stored position. \\\n",
    "            Used as the \"candidate\" in the stored CSV\n",
    "        '''\n",
    "\n",
    "        return str(self.pos).replace(' ', '')\n",
    "\n",
    "\n",
    "    # --- Setters & Getters --- #\n",
    "\n",
    "    @property\n",
    "    def position(self):\n",
    "        '''The :code:`position` attribute getter\n",
    "\n",
    "        Returns:\n",
    "            str: the stored position\n",
    "        '''\n",
    "\n",
    "        return self.pos\n",
    "\n",
    "\n",
    "    @property\n",
    "    def fitness(self):\n",
    "        '''The :code:`fitness` attribute getter\n",
    "\n",
    "        Returns:\n",
    "            float: the stored fitness value\n",
    "        '''\n",
    "\n",
    "        return self.fit\n",
    "\n",
    "\n",
    "    @property\n",
    "    def time(self):\n",
    "        '''The :code:`eval_time` attribute getter\n",
    "\n",
    "        Returns:\n",
    "            float: the stored evaluation time value (in seconds)\n",
    "        '''\n",
    "\n",
    "        return self.eval_time\n",
    "\n",
    "\n",
    "    @position.setter\n",
    "    def position(self, value):\n",
    "        '''The :code:`position` attribute setter\n",
    "\n",
    "        Args:\n",
    "            value (str): new position value to set\n",
    "        '''\n",
    "\n",
    "        self.pos = value\n",
    "\n",
    "\n",
    "    @fitness.setter\n",
    "    def fitness(self, value):\n",
    "        '''The :code:`fittness` attribute setter\n",
    "\n",
    "        Args:\n",
    "            value (float): new fitness value to set\n",
    "        '''\n",
    "\n",
    "        self.fit = value\n",
    "\n",
    "\n",
    "    @time.setter\n",
    "    def time(self, value):\n",
    "        '''The :code:`eval_time` attribute setter\n",
    "\n",
    "        Args:\n",
    "            value (float): new evaluation time value (in seconds) to set\n",
    "        '''\n",
    "\n",
    "        self.eval_time = value\n",
    "\n",
    "\n",
    "    def __str__(self):\n",
    "        '''For logging/debugging purposes\n",
    "\n",
    "        Returns:\n",
    "            str: the pretty-print contents of the FoodSource\n",
    "        '''\n",
    "\n",
    "        return 'position: {}, fitness: {}, evaluation time: {}'.format(self.pos,\n",
    "                                                                       self.fit,\n",
    "                                                                       self.eval_time)\n",
    "\n",
    "\n",
    "    def __repr__(self):\n",
    "        '''For logging/debugging purposes\n",
    "\n",
    "        Returns:\n",
    "            str: the pretty-print contents of the FoodSource (forced as str)\n",
    "        '''\n",
    "\n",
    "        return str(self)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3uqPXJlagRWu"
   },
   "source": [
    "#### ArtificialBee Base Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "aPkKWP7ow5Gq"
   },
   "outputs": [],
   "source": [
    "\"\"\"Abstract definitions of the\n",
    ":class:`~core.abc.employee_bee.EmployeeBee` and\n",
    ":class:`~core.abc.onlooker_bee.OnlookerBee` methods\n",
    "\"\"\"\n",
    "\n",
    "class ArtificialBee(ABC):\n",
    "    '''Abstract class for Employee & Onlooker Bees\n",
    "\n",
    "    Attributes:\n",
    "        food_source (:class:`~core.abc.food_sourec.FoodSource`): the bee's main \\\n",
    "        food source\n",
    "        id (int): the bee's ID, used for logging/tracking purposes\n",
    "    '''\n",
    "\n",
    "    def __init__(self, food_source, id):\n",
    "        '''Initialize an Artificial Bee\n",
    "\n",
    "        Args:\n",
    "            food_source (:class:`~core.abc.food_sourec.FoodSource`): the bee's \\\n",
    "            main food source\n",
    "            id (int): the bee's ID, used for logging/tracking purposes\n",
    "        '''\n",
    "\n",
    "        self.food_source = food_source\n",
    "        self.id = id if id is not None else -1\n",
    "\n",
    "\n",
    "    def get_random_neighbor(self, obj_interface: ObjectiveInterface):\n",
    "        '''\n",
    "            Finds a random neighbor in the vicinity of the parent\n",
    "            Parent(Onlooker) = Employee,\n",
    "            Parent(Employee) = Scout\n",
    "            Given by [1]:\n",
    "                Xmi = Li + rand(0, 1) ∗ (Ui − Li)   => Initial FoodSource\n",
    "                                                       (Scout)\n",
    "                υmi = Xmi + ϕmi(Xmi − Xki)          => Neighboring FoodSource\n",
    "                                                       (Employee/Onlooker)\n",
    "                Where υmi is a neighboring FoodSource\n",
    "                (definition of \"neighboring\" given in [2];\n",
    "\n",
    "                TLDR - in numerical and continuous optimization problems,\n",
    "                a dimensional component is incremented/decremented.\n",
    "                In NAS context, it is a 1-operation difference per network)\n",
    "\n",
    "            [1] Karaboga, D., & Basturk, B. (2007). A powerful and efficient\n",
    "            algorithm for numerical function optimization: artificial bee\n",
    "            colony (ABC) algorithm. Journal of global optimization, 39(3),\n",
    "            459-471.\n",
    "            [2] White, C., Nolen, S., & Savani, Y. (2021, December).\n",
    "            Exploring the loss landscape in neural architecture search.\n",
    "            In Uncertainty in Artificial Intelligence (pp. 654-664). PMLR.\n",
    "\n",
    "\n",
    "        Args:\n",
    "            obj_interface (:class:`~core.objective_interface.ObjectiveInterface`): the objective interface \\\n",
    "            used to sample candidates\n",
    "\n",
    "        Returns:\n",
    "            :class:`~core.abc.food_source.FoodSource`: a randomly sampled neighboring food source\n",
    "        '''\n",
    "\n",
    "        pos = self.get_center_fs().position\n",
    "        neighbor_pos = obj_interface.get_neighbor(pos)\n",
    "\n",
    "        return FoodSource(neighbor_pos)\n",
    "\n",
    "\n",
    "    def is_evaluated(self):\n",
    "        '''\n",
    "        Checks if food source is evaluated for solution tracking purposes\n",
    "\n",
    "        Returns:\n",
    "            bool: whether or not the current :class:`~core.abc.food_source.FoodSource` \\\n",
    "            is evaluated\n",
    "        '''\n",
    "\n",
    "        return self.food_source is not None and \\\n",
    "        self.food_source.fitness is not None\n",
    "\n",
    "\n",
    "    @abstractmethod\n",
    "    def get_center_fs(self):\n",
    "        '''Returns the center food source\n",
    "\n",
    "        Returns:\n",
    "            :class:`~core.abc.food_source.FoodSource`: the employee's center food source\n",
    "\n",
    "        Raises:\n",
    "            :class:`NotImplementedError`: must be implemented by the child class\n",
    "        '''\n",
    "\n",
    "        raise NotImplementedError()\n",
    "\n",
    "\n",
    "    @abstractmethod\n",
    "    def evaluate(self, obj_interface: ObjectiveInterface):\n",
    "        '''Evaluates the current :class:`core.abc.food_source.FoodSource`\n",
    "\n",
    "        Args:\n",
    "            obj_interface (:class:`~core.objective_interface.ObjectiveInterface`): the objective interface \\\n",
    "            used to sample/evaluate candidates\n",
    "\n",
    "        Returns:\n",
    "            :class:`pandas.Series`: a Pandas Series containing the evaluation's results (represents a \\\n",
    "            row in the main results CSV file)\n",
    "\n",
    "        Raises:\n",
    "            :class:`NotImplementedError`: must be implemented by the child class\n",
    "        '''\n",
    "\n",
    "        raise NotImplementedError()\n",
    "\n",
    "\n",
    "    @abstractmethod\n",
    "    def search(self, obj_interface: ObjectiveInterface):\n",
    "        \"\"\"Explore new random position (near previously-sampled position) and assigns it to the \\\n",
    "        current food source\n",
    "\n",
    "        Args:\n",
    "            obj_interface (:class:`~core.objective_interface.ObjectiveInterface`): the objective interface \\\n",
    "            used to sample/evaluate candidates\n",
    "\n",
    "        Raises:\n",
    "            :class:`NotImplementedError`: must be implemented by the child class\n",
    "        \"\"\"\n",
    "\n",
    "        raise NotImplementedError()\n",
    "\n",
    "\n",
    "    def __repr__(self):\n",
    "        '''For logging/debugging purposes\n",
    "\n",
    "        Returns:\n",
    "            str: the pretty-print contents of the Employee/Onlooker Bee\n",
    "        '''\n",
    "\n",
    "        return str(self)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "G1BmrcYbgV7w"
   },
   "source": [
    "#### EmployeeBee"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "qjZG9UsKxGjD"
   },
   "outputs": [],
   "source": [
    "\"\"\"Employee Bees' class responsible for the exploration phase of the\n",
    "Artificial Bee Colony optimization\n",
    "\"\"\"\n",
    "\n",
    "class EmployeeBee(ArtificialBee):\n",
    "    '''Employee Bees, responsible for explorations and partial exploitation of the\n",
    "    solution space. Employees search for food sources in the neighborhood, evaluate\n",
    "    candidates, and compute the probabilities needed for the stochastic onlooker assignment\n",
    "\n",
    "    Attributes:\n",
    "        center_fs (:class:`~core.abc.food_source.FoodSource`): the central food souorce \\\n",
    "        which can be greedy-selected by associated onlookers during exploitation. This \\\n",
    "        food source holds the best fitness in the evaluated vicinity (neighbors)\n",
    "        food_source (:class:`~core.abc.food_source.FoodSource`): the employee's current \\\n",
    "        food source (i.e during non-initial iterations when employees are exploiting)\n",
    "        id_tracker (int): the bee's ID for logging and tracking purposes\n",
    "        trials (int): the number of trials/evaluations done around a given center \\\n",
    "        food source. Used to abandon an area once the abandonment limit is reached\n",
    "    '''\n",
    "\n",
    "    def __init__(self, food_source):\n",
    "        '''Initializes an employee bee with a center food source\n",
    "\n",
    "        Args:\n",
    "            food_source (:class:`~core.abc.food_source.FoodSource`): the initial FoodSource to \\\n",
    "            be assigned as the :code:`center_fs`\n",
    "        '''\n",
    "\n",
    "        if not hasattr(EmployeeBee, 'id_tracker'):\n",
    "            EmployeeBee.id_tracker = 0\n",
    "\n",
    "        super(EmployeeBee, self).__init__(food_source, EmployeeBee.id_tracker)\n",
    "\n",
    "        EmployeeBee.id_tracker += 1\n",
    "        self.trials = 0\n",
    "        self.center_fs = food_source   # center_fs can be greedy-selected by\n",
    "                                       # child onlookers (i.e the new\n",
    "                                       # optimization center)\n",
    "\n",
    "\n",
    "    def search(self, obj_interface):\n",
    "        '''\n",
    "        Explore new random position (near previously-sampled position) and assigns it to the \\\n",
    "        current food source\n",
    "\n",
    "        Args:\n",
    "            obj_interface (:class:`~core.objective_interface.ObjectiveInterface`): the given \\\n",
    "            objective interface used to sample/evaluate candidates\n",
    "        '''\n",
    "\n",
    "        if self.food_source.fitness is None:\n",
    "            # unevaluated (first iteration after abandonment reset)\n",
    "            return\n",
    "\n",
    "        # find neighbor near food_source in the bee's memory\n",
    "        self.food_source = self.get_random_neighbor(obj_interface)\n",
    "\n",
    "\n",
    "    def reset(self, new_fs):\n",
    "        '''Resets EmployeeBee once abandonment limit is reached\n",
    "\n",
    "        Args:\n",
    "            new_fs (:code:`core.abc.food_source.FoodSource`): a reintialization food source. \\\n",
    "            Assigned as the :code:`center_fs`\n",
    "        '''\n",
    "\n",
    "        self.trials = 0\n",
    "        self.food_source = new_fs\n",
    "        self.center_fs = new_fs\n",
    "\n",
    "\n",
    "    def calculate_fitness(self):\n",
    "        '''Calculate fitness of an :class:`~core.abc.employee_bee.EmployeeBee`, given by:\n",
    "\n",
    "                         ⎧ 1 / (1 + Fm(Xm→))       if  Fm(Xm→)≥0\n",
    "            Fit_m(Xm→)=  ⎨\n",
    "                         ⎩ 1 + abs(Fm(Xm→))        if  Fm(Xm→)<0\n",
    "\n",
    "        Returns:\n",
    "            float: adjusted fitness value for the stochastic assignment operator\n",
    "        '''\n",
    "\n",
    "        fitm = 0\n",
    "        if self.center_fs.fitness >= 0:\n",
    "            fitm = 1 / (1 + self.center_fs.fitness)\n",
    "        else:\n",
    "            fitm = 1 + np.abs(self.center_fs.fitness)\n",
    "\n",
    "        return fitm\n",
    "\n",
    "\n",
    "    def compute_probability(self, sum_fitness):\n",
    "        '''\n",
    "        Calculate probability of an EmployeeBee being chosen by\n",
    "        an OnlookerBee based on Fitess values; given by:\n",
    "\n",
    "        Pn = Fit_n(Xn→) / [∑ (Fit_m(Xm→)) for all m]\n",
    "\n",
    "        Args:\n",
    "            sum_fitness (float): sum of all fitness values in the population, used for the roulette wheel selector\n",
    "\n",
    "        Returns:\n",
    "            float: calculated probability that the current employee should be selected by an onlooker\n",
    "        '''\n",
    "\n",
    "        return self.calculate_fitness() / sum_fitness\n",
    "\n",
    "\n",
    "    def evaluate(self, obj_interface, itr):\n",
    "        '''Evaluates sampled position and increments trial counter\n",
    "\n",
    "        Args:\n",
    "            obj_interface (:class:`~core.objective_interface.ObjectiveInterface`): the objective interface \\\n",
    "            used to sample/evaluate candidates\n",
    "            itr (int): current ABC iteration (for logging and result-saving purposes)\n",
    "\n",
    "        Returns:\n",
    "            :class:`pandas.Series`: a Pandas Series containing the evaluation's results (represents a \\\n",
    "            row in the main results CSV file)\n",
    "        '''\n",
    "\n",
    "        Logger.evaluation_log('EmployeeBee', self.id, self.food_source.position)\n",
    "        t = time.time()\n",
    "\n",
    "        res = obj_interface.evaluate(self.food_source.position)\n",
    "        # unpack\n",
    "        self.food_source.fitness = res['fitness']\n",
    "        epochs = res['epochs']\n",
    "        weights_filename = res['filename']\n",
    "        params = res['params']\n",
    "        momentum = res['momentum']\n",
    "\n",
    "        self.food_source.time = time.time() - t\n",
    "        # ACT early bandonment (ACT enabled and network could not pass epoch 1)\n",
    "        abandon_early = Params['TERMINATION_THRESHOLD_FACTOR'] > 0.0 and epochs <= 1\n",
    "        self.trials += 1 if not abandon_early else Params['ABANDONMENT_LIMIT']\n",
    "\n",
    "        if self.center_fs.fitness is None:\n",
    "            # Check if this evaluation is the first in its area\n",
    "            # (Iteration 1 after reset; i.e no need to greedy-select)\n",
    "            self.center_fs = deepcopy(self.food_source)\n",
    "        else:\n",
    "            # Greedy select for iterations 2 ... Abandonment Limit\n",
    "            self.greedy_select(self.food_source, obj_interface.is_minimize)\n",
    "\n",
    "        # save data\n",
    "        series = pd.Series({\n",
    "            'bee_type': type(self).__name__,\n",
    "            'bee_id': self.id,\n",
    "            'bee_parent': '-',\n",
    "            'itr': itr,\n",
    "            'candidate': self.food_source.position,\n",
    "            'fitness': self.food_source.fitness,\n",
    "            'center_fitness': self.center_fs.fitness,\n",
    "            'momentum': sum([x[1] for _,x in momentum.items()]),\n",
    "            'epochs': epochs,\n",
    "            'momentum_epochs': 0,\n",
    "            'params': params,\n",
    "            'weights_filename': weights_filename,\n",
    "            'time': self.food_source.time\n",
    "        })\n",
    "        return series\n",
    "\n",
    "\n",
    "    def greedy_select(self, n_food_source, is_minimize):\n",
    "        '''Update best FoodSource to minimize or maximize fitness (elitism)\n",
    "\n",
    "        Args:\n",
    "            n_food_source (:class:`~core.abc.food_source.FoodSource`): the new food source to \\\n",
    "            be greedy-selected\n",
    "            is_minimize (bool): determines whether to minimize or maximize the greedy-selection\n",
    "        '''\n",
    "\n",
    "        if ((self.center_fs.fitness < n_food_source.fitness) and not is_minimize) or \\\n",
    "        ((self.center_fs.fitness > n_food_source.fitness) and is_minimize):\n",
    "            self.center_fs.position = n_food_source.position\n",
    "            self.center_fs.fitness = n_food_source.fitness\n",
    "\n",
    "\n",
    "    def get_center_fs(self):\n",
    "        '''Returns the center food source\n",
    "\n",
    "        Returns:\n",
    "            :class:`~core.abc.food_source.FoodSource`: the employee's center food source\n",
    "        '''\n",
    "\n",
    "        return self.center_fs\n",
    "\n",
    "\n",
    "    def __str__(self):\n",
    "        '''For logging/debugging purposes\n",
    "\n",
    "        Returns:\n",
    "            str: the pretty-print contents of the EmployeeBee\n",
    "        '''\n",
    "\n",
    "        return 'EmployeeBee {} -- FS: {}, trials: {}'.format(self.id, \\\n",
    "                                                             self.food_source, \\\n",
    "                                                             self.trials)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "adQwk052ga1q"
   },
   "source": [
    "#### OnlookerBee"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "cIwN0V4PxDQX"
   },
   "outputs": [],
   "source": [
    "\"\"\"Onlooker Bees' class responsible for the exploitation phase of the\n",
    "Artificial Bee Colony optimization\n",
    "\"\"\"\n",
    "\n",
    "class OnlookerBee(ArtificialBee):\n",
    "    '''Onlooker Bees, primarily responsible for the exploitation of \\\n",
    "    the solution space. Onlookers search for neighboring food sources.\n",
    "\n",
    "    Attributes:\n",
    "        employee (:class:`~core.abc.employee_bee.EmployeeBee`): the assigned \\\n",
    "        Employee Bee; stored to access greedy-selection\n",
    "        food_source (:class:`~core.abc.food_source.FoodSource`): the current \\\n",
    "        food source (i.e neighbor)\n",
    "        id_tracker (int): the bee's ID for logging and tracking purposes\n",
    "    '''\n",
    "\n",
    "    def __init__(self):\n",
    "        '''Initializes an onlooker bee\n",
    "        '''\n",
    "\n",
    "        if not hasattr(OnlookerBee, 'id_tracker'):\n",
    "            OnlookerBee.id_tracker = 0\n",
    "\n",
    "        super(OnlookerBee, self).__init__(None, OnlookerBee.id_tracker)\n",
    "\n",
    "        OnlookerBee.id_tracker += 1\n",
    "        self.employee = None\n",
    "\n",
    "\n",
    "    def search(self, obj_interface):\n",
    "        '''\n",
    "        Exploit position (near a random :class:`~core.abc.employee_bee.EmployeeBee` \\\n",
    "        chosen according to computed probability)\n",
    "\n",
    "        Args:\n",
    "            obj_interface (:class:`~core.objective_interface.ObjectiveInterface`): the \\\n",
    "            objective interface used to sample/evaluate candidates\n",
    "        '''\n",
    "\n",
    "        self.food_source = self.get_random_neighbor(obj_interface)\n",
    "\n",
    "\n",
    "    def assign_employee(self, employee):\n",
    "        '''Assigns an EmployeeBee to the Onlooker for neighbor-search\n",
    "\n",
    "        Args:\n",
    "            employee (:class:`~core.abc.employee_bee.EmployeeBee`): the \\\n",
    "            assigned employee bee\n",
    "        '''\n",
    "\n",
    "        self.employee = employee\n",
    "        self.food_source = FoodSource(self.employee.food_source.position)\n",
    "\n",
    "\n",
    "    def evaluate(self, obj_interface, itr):\n",
    "        '''\n",
    "        Evaluates sampled position and increments employee's trial counter\n",
    "\n",
    "        Args:\n",
    "            obj_interface (:class:`~core.objective_interface.ObjectiveInterface`): the objective interface \\\n",
    "            used to sample/evaluate candidates\n",
    "            itr (int): current ABC iteration (for logging and result-saving purposes)\n",
    "\n",
    "        Returns:\n",
    "            :class:`pandas.Series`: a Pandas Series containing the evaluation's results (represents a \\\n",
    "            row in the main results CSV file)\n",
    "        '''\n",
    "\n",
    "        Logger.evaluation_log('OnlookerBee', self.id, self.food_source.position)\n",
    "        t = time.time()\n",
    "\n",
    "        res = obj_interface.evaluate(self.food_source.position)\n",
    "        # unpack\n",
    "        self.food_source.fitness = res['fitness']\n",
    "        epochs = res['epochs']\n",
    "        weights_filename = res['filename']\n",
    "        params = res['params']\n",
    "        momentum = res['momentum']\n",
    "\n",
    "        self.food_source.time = time.time() - t\n",
    "        self.employee.trials += 1\n",
    "        self.employee.greedy_select(self.food_source, obj_interface.is_minimize)\n",
    "\n",
    "        series = pd.Series({\n",
    "            'bee_type': type(self).__name__,\n",
    "            'bee_id': self.id,\n",
    "            'bee_parent': self.employee.id,\n",
    "            'itr': itr,\n",
    "            'candidate': self.food_source.position,\n",
    "            'fitness': self.food_source.fitness,\n",
    "            'center_fitness': self.get_center_fs().fitness,\n",
    "            'momentum': sum([x[1] for _,x in momentum.items()]) / len(momentum),\n",
    "            'epochs': epochs,\n",
    "            'momentum_epochs': 0,\n",
    "            'params': params,\n",
    "            'weights_filename': weights_filename,\n",
    "            'time': self.food_source.time\n",
    "        })\n",
    "        return series\n",
    "\n",
    "\n",
    "    def get_center_fs(self):\n",
    "        '''Returns the parent's center food_source\n",
    "\n",
    "        Returns:\n",
    "            :class:`~core.abc.food_source.FoodSource`: associated employee bee's \\\n",
    "            center food source\n",
    "        '''\n",
    "\n",
    "        return self.employee.center_fs\n",
    "\n",
    "\n",
    "    def __str__(self):\n",
    "        '''For logging/debugging purposes\n",
    "\n",
    "        Returns:\n",
    "            str: the pretty-print contents of the OnlookerBee\n",
    "        '''\n",
    "\n",
    "        return 'OnlookerBee {} -> Parent Employee ({}) -- FS: {}'\\\n",
    "        .format(self.id, self.employee.id, self.food_source)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ZibiU5kTgd5U"
   },
   "source": [
    "#### ScoutBee"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "WxC8POuNyH9U"
   },
   "outputs": [],
   "source": [
    "\"\"\"ABC's Scout Bee module\n",
    "\"\"\"\n",
    "\n",
    "class ScoutBee:\n",
    "    '''Scout Bees' static methods.\n",
    "\n",
    "    ABC Scout Bees are generally classified as a *reset* operator\n",
    "    for Employee/Onlooker Bees.\n",
    "\n",
    "    Responsible for sampling the initial random FoodSources Xm→\n",
    "    and reseting employee bees when the abandonment limit is reached\n",
    "    '''\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def sample(obj_interface):\n",
    "        '''Sample a random point from the objective function\n",
    "\n",
    "        Args:\n",
    "            obj_interface (:class:`~core.objective_interface.ObjectiveInterface`): the objective interface that defines the boundaries of the problem\n",
    "\n",
    "        Returns:\n",
    "            :class:`~core.abc.food_source.FoodSource`: the randomly sampled initial position\n",
    "        '''\n",
    "\n",
    "        return FoodSource(obj_interface.sample())\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def check_abandonment(employees, obj_interface):\n",
    "        '''Check if EmployeeBees reached abandonment limit\n",
    "\n",
    "        Args:\n",
    "            employees (list): a list of :class:`~core.abc.employee_bee.EmployeeBee` to check their trial count / abandonment limit\n",
    "            obj_interface (:class:`~core.objective_interface.ObjectiveInterface`): the objective interface, needed to :func:`core.abc.scout_bee.ScoutBee.sample` a new position if the abandonment limit is reached\n",
    "        '''\n",
    "\n",
    "        for employee in employees:\n",
    "            if employee.trials >= Params['ABANDONMENT_LIMIT']:\n",
    "                # Reset EmployeeBees to a new ScoutBee position\n",
    "                employee.reset(ScoutBee.sample(obj_interface))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ufPCYNFeggVR"
   },
   "source": [
    "#### ABC Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4zWzCTChHSOr"
   },
   "outputs": [],
   "source": [
    "\"\"\"The main Artificial Bee Colony optimization algorithm\n",
    "\"\"\"\n",
    "\n",
    "class ArtificialBeeColony:\n",
    "    '''Artificial Bee Colony optimizer\n",
    "\n",
    "    Attributes:\n",
    "        colony_size (int, optional): bee population size; defaults to value set in \\\n",
    "        :class:`~config.params.Params`\n",
    "        employees (list): list of all :class:`~core.abc.employee_bee.EmployeeBee` used in the \\\n",
    "        optimization\n",
    "        eo_colony_ratio (float, optional): employees to onlookers ratio; defaults to value set in \\\n",
    "        :class:`~config.params.Params`\n",
    "        obj_interface (:class:`~core.objective_interface.ObjectiveInterface`): the objective \\\n",
    "        interface defining the task boundaries\n",
    "        onlookers (list): list of all :class:`~core.abc.onlooker_bee.OnlookerBee` used in the \\\n",
    "        optimization\n",
    "        results_df (:class:`pandas.DataFrame`): the main results DataFrame containing all evaluated \\\n",
    "        results\n",
    "        scouts (list): list of all :class:`~core.abc.scout_bee.ScoutBee`-sampled positions used in  \\\n",
    "        the optimization. Scout Bees are *not* instantiated.\n",
    "        scouts_count (int): number of scouts / parallel explorations. Follows the classical ABC \\\n",
    "        1-to-1 scout-to-employee ratio\n",
    "        total_evals (int): total number of :func:`~core.abc.artificial_bee.ArtificialBee.evaluate` \\\n",
    "        calls (for both Employees and Onlookers)\n",
    "    '''\n",
    "\n",
    "    def __init__(self, obj_interface, \\\n",
    "                 colony_size=Params['COLONY_SIZE'], \\\n",
    "                 employee_onlooker_ratio=Params['EMPLOYEE_ONLOOKER_RATIO']):\n",
    "        ''' Initializes the ABC algorithm '''\n",
    "\n",
    "        self.obj_interface = obj_interface    # Encapsulates the Search Space +\n",
    "                                              # Evaluation Strategy\n",
    "        self.colony_size = colony_size\n",
    "        self.eo_colony_ratio = employee_onlooker_ratio\n",
    "        self.scouts_count = int(self.colony_size * self.eo_colony_ratio)\n",
    "\n",
    "\n",
    "    def __init_scouts(self):\n",
    "        '''Initial :class:`~core.abc.food_source.FoodSource` sampling by Scout Bees \\\n",
    "        *(does not evaluate fitness)*\n",
    "        '''\n",
    "\n",
    "        for _ in range(self.scouts_count):\n",
    "            self.scouts.append(ScoutBee.sample(self.obj_interface))\n",
    "\n",
    "\n",
    "    def __init_employees(self):\n",
    "        ''' Instantiate Employee Bees and assign a Scout position to each '''\n",
    "\n",
    "        # Floor of (colony_size * ratio)\n",
    "        employee_count = int(self.colony_size * self.eo_colony_ratio)\n",
    "\n",
    "        for itr in range(employee_count):\n",
    "            # Split scouts evenly among employees\n",
    "            scout = self.scouts[int(itr / (employee_count / \\\n",
    "                                           self.scouts_count))]\n",
    "            self.employees.append(EmployeeBee(scout))\n",
    "\n",
    "\n",
    "    def __init_onlookers(self):\n",
    "        '''Instantiate Onlooker Bees *(assigning Employees occurs after \\\n",
    "        evaluation and probability calculation)*\n",
    "        '''\n",
    "\n",
    "        onlooker_count = self.colony_size - int(self.colony_size * self.eo_colony_ratio)\n",
    "\n",
    "        for itr in range(onlooker_count):\n",
    "            self.onlookers.append(OnlookerBee())\n",
    "\n",
    "\n",
    "    def __employee_bee_phase(self, itr):\n",
    "        '''Evaluate Scout-initialized position after :func:`~core.abc.scout_bee.ScoutBee.reset` \\\n",
    "        or Search + Evaluate neighbor every subsequent iteration \\\n",
    "        until abandonment limit\n",
    "\n",
    "        Args:\n",
    "            itr (int): current main ABC optimization iteration\n",
    "        '''\n",
    "\n",
    "        # Search and evaluate new or existing neighbor\n",
    "        for employee in self.employees:\n",
    "            # Ignored if unevaluated position exists (i.e Scout-sampled)\n",
    "            employee.search(self.obj_interface)\n",
    "\n",
    "            fs = employee.food_source\n",
    "\n",
    "            if fs is None or fs.encode_position() not in self.results_df['candidate']:\n",
    "                # Evaluate employee position\n",
    "                series = employee.evaluate(self.obj_interface, itr)\n",
    "                self.__save_results(series)\n",
    "            else:\n",
    "                # Already evaluated\n",
    "                fs.fitness = self.results_df[self.results_df['candidate'] == fs.encode_position()]['fitness'].values[0]\n",
    "                employee.greedy_select(fs, self.obj_interface.is_minimize)\n",
    "                # resampling the same candidate should count as a trial towards the abandonment limit?\n",
    "                # onlooker.employee.trials += 1     # to avoid being stuck\n",
    "\n",
    "\n",
    "    def __onlooker_bee_phase(self, itr):\n",
    "        '''Assign each Onlooker to an Employee, then Search + Evaluate a random neighbor\n",
    "\n",
    "        Args:\n",
    "            itr (int): current main ABC optimization iteration\n",
    "        '''\n",
    "\n",
    "        # Calculate Employee probability\n",
    "        sum_fitness = sum([employee.calculate_fitness() \\\n",
    "                        for employee in self.employees])\n",
    "\n",
    "        # Sum(probabilities) ≈ 1.0\n",
    "        probabilities = list(map(lambda employee: \\\n",
    "                                 employee.compute_probability(sum_fitness), \\\n",
    "                                 self.employees))\n",
    "\n",
    "        if not self.obj_interface.is_minimize:\n",
    "            # inverse weights to maximize the objective\n",
    "            # (assign more onlookers to higher fitness scores)\n",
    "\n",
    "            # reciprocals\n",
    "            probabilities = [1.0 / prob for prob in probabilities]\n",
    "\n",
    "            # normalize\n",
    "            sum_probs = sum(probabilities)\n",
    "            probabilities = [prob / sum_probs for prob in probabilities]\n",
    "\n",
    "        # Assign EmployeeBees to OnlookerBees, search, and evaluate neighbor\n",
    "        for onlooker in self.onlookers:\n",
    "            # Assign EmployeeBees to OnlookerBees\n",
    "            emp_idx = np.random.choice(len(self.employees), p=probabilities)\n",
    "\n",
    "            onlooker.assign_employee(self.employees[emp_idx])\n",
    "\n",
    "            # Search for a new random neighbor\n",
    "            onlooker.search(self.obj_interface)\n",
    "\n",
    "            fs = onlooker.food_source\n",
    "\n",
    "            if fs is None or fs.encode_position() not in self.results_df['candidate']:\n",
    "                # Evaluate employee position\n",
    "                series = onlooker.evaluate(self.obj_interface, itr)\n",
    "                self.__save_results(series)\n",
    "            else:\n",
    "                # Already evaluated\n",
    "                fs.fitness = self.results_df[self.results_df['candidate'] == fs.encode_position()]['fitness'].values[0]\n",
    "                onlooker.employee.greedy_select(fs, self.obj_interface.is_minimize)\n",
    "                # resampling the same candidate should count as a trial towards the abandonment limit?\n",
    "                # onlooker.employee.trials += 1     # to avoid being stuck\n",
    "\n",
    "\n",
    "    def __scout_bee_phase(self):\n",
    "        '''Check abandonment limits and rest employees accordingly\n",
    "        '''\n",
    "\n",
    "        ScoutBee.check_abandonment(self.employees, self.obj_interface)\n",
    "\n",
    "\n",
    "    def __save_results(self, series):\n",
    "        '''Save results dataframe to specified disk path\n",
    "\n",
    "        Args:\n",
    "            series (:class:`pandas.Series): Pandas Series (row) to be appended to the \\\n",
    "            current :code:`results_df`\n",
    "        '''\n",
    "\n",
    "        if self.total_evals % Params['RESULTS_SAVE_FREQUENCY'] == 0:\n",
    "            self.results_df = self.results_df.append(series, ignore_index=True)\n",
    "\n",
    "            filename = f'{Params[\"CONFIG_VERSION\"]}.csv'\n",
    "            if FileHandler.save_df(self.results_df,\n",
    "                                   Params.get_results_path(),\n",
    "                                   filename):\n",
    "                Logger.filesave_log(series['candidate'], series['weights_filename'])\n",
    "\n",
    "\n",
    "    def __momentum_phase(self):\n",
    "        '''Momentum Evaluation Augmentation Stochastic operator that adds \\\n",
    "        :code:`Params['MOMENTUM_EPOCHS']` epochs to propel the evaluations of \\\n",
    "        the most consistently converging candidates\n",
    "        '''\n",
    "\n",
    "\n",
    "        # calculate probabilities\n",
    "        calculated_momentums = self.results_df['momentum'] / (self.results_df['epochs'] + \\\n",
    "                                                              self.results_df['momentum_epochs'])\n",
    "        if sum(calculated_momentums) == 0:\n",
    "            # improbable edge case where the sum of momentums is exactly 0\n",
    "            return\n",
    "\n",
    "        probs = calculated_momentums / sum(calculated_momentums)\n",
    "\n",
    "        the_chosen_ones = dict.fromkeys([x for x in range(len(probs))], 0)\n",
    "\n",
    "        for _ in range(Params['MOMENTUM_EPOCHS']):\n",
    "            # probabilistically assign momentum epochs\n",
    "            idx = np.random.choice(len(probs), p=probs)\n",
    "            the_chosen_ones[idx] += 1\n",
    "\n",
    "        # training extension loop\n",
    "        for the_one, m_epochs in the_chosen_ones.items():\n",
    "            candidate_row = self.results_df.iloc[[the_one]]\n",
    "\n",
    "            # extract candidate info for additional training\n",
    "            candidate = candidate_row['candidate'].values[0]\n",
    "            weights_file = candidate_row['weights_filename'].values[0]\n",
    "            momentum = candidate_row['momentum'].values[0]\n",
    "            epochs = candidate_row['epochs'].values[0]\n",
    "            momentum_epochs = candidate_row['momentum_epochs'].values[0] + m_epochs\n",
    "\n",
    "            # train for m_epochs\n",
    "            Logger.momentum_evaluation_log(candidate,\n",
    "                                           candidate_row['fitness'].values[0],\n",
    "                                           m_epochs)\n",
    "\n",
    "            res = self.obj_interface.momentum_eval(candidate,\n",
    "                                                   weights_file,\n",
    "                                                   m_epochs)\n",
    "\n",
    "            # save new results\n",
    "            self.results_df.loc[the_one, 'fitness'] = res['fitness']\n",
    "            self.results_df.loc[the_one, 'momentum_epochs'] = momentum_epochs\n",
    "\n",
    "\n",
    "    def __reset_all(self):\n",
    "        ''' Resets the ABC algorithm\n",
    "        '''\n",
    "\n",
    "        self.scouts = []            # List of FoodSources initially sampled\n",
    "        self.employees = []\n",
    "        self.onlookers = []\n",
    "\n",
    "        EmployeeBee.id_tracker = 0\n",
    "        OnlookerBee.id_tracker = 0\n",
    "\n",
    "        # init results dataframe\n",
    "        filename = f'_{Params[\"CONFIG_VERSION\"]}.csv'\n",
    "        results_file = os.path.join(Params.get_results_path(), filename)\n",
    "\n",
    "        # resume from previously saved file if it exists\n",
    "        if Params['RESUME_FROM_RESULTS_FILE']:\n",
    "            self.results_df = FileHandler.load_df(results_file)     # loads empty df if file not found\n",
    "\n",
    "        else:\n",
    "            cols = ['bee_type'\n",
    "            'bee_id',\n",
    "            'bee_parent',\n",
    "            'itr',\n",
    "            'candidate',\n",
    "            'fitness',\n",
    "            'center_fitness',\n",
    "            'epochs',\n",
    "            'params',\n",
    "            'weights_filename',\n",
    "            'time']\n",
    "\n",
    "            self.results_df = pd.DataFrame(columns=cols)\n",
    "\n",
    "        self.total_evals = len(self.results_df.index)\n",
    "\n",
    "\n",
    "    def optimize(self):\n",
    "        ''' Main optimization loop\n",
    "        '''\n",
    "\n",
    "        Params.export_yaml(Params.get_results_path(),\n",
    "                           f'{Params[\"CONFIG_VERSION\"]}.yaml')\n",
    "\n",
    "        Logger.start_log()\n",
    "\n",
    "        self.__reset_all()\n",
    "        self.__init_scouts()\n",
    "        self.__init_employees()\n",
    "        self.__init_onlookers()\n",
    "        start_time = time.time()\n",
    "\n",
    "        fitness_selector = max if not self.obj_interface.is_minimize else min\n",
    "\n",
    "        ''' Optimization loop '''\n",
    "        for itr in range(Params['ITERATIONS_COUNT']):\n",
    "            self.__employee_bee_phase(itr)\n",
    "            self.__onlooker_bee_phase(itr)\n",
    "            self.__momentum_phase()\n",
    "            self.__scout_bee_phase()\n",
    "\n",
    "            best_fitness = fitness_selector(self.results_df['fitness'].tolist())\n",
    "\n",
    "            if itr % 1 == 0:\n",
    "                Logger.status(itr,\n",
    "                          'Best fitness: {}, Total time (s): {}'.format(best_fitness,\n",
    "                                                                        time.time() - start_time))\n",
    "\n",
    "        Logger.end_log()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "KBY5aOEZgo0Z"
   },
   "source": [
    "### HiveNAS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "TxYhgNtsJW7i"
   },
   "outputs": [],
   "source": [
    "\"\"\"Top-level module used to run HiveNAS\n",
    "\"\"\"\n",
    "\n",
    "class HiveNAS(object):\n",
    "    '''Encapsulates all high level modules and runs the \\\n",
    "    ABC-based optimization\n",
    "    '''\n",
    "\n",
    "    @staticmethod\n",
    "    def find_topology(evaluation_logging=True,\n",
    "                      config_path=None,\n",
    "                      kill_after=True):\n",
    "        '''Runs the base NAS optimization loop\n",
    "\n",
    "        Args:\n",
    "            evaluation_logging (bool, optional): determines whether to log \\\n",
    "            evaluation info or not; defaults to :code:`True`\n",
    "            config_path (str, optional): yaml configuration file path; \\\n",
    "            defaults to hard-coded config in :class:`~config.params.Params`\n",
    "        '''\n",
    "\n",
    "        if config_path:\n",
    "            # load yaml config from given path\n",
    "            Params.init_from_yaml(config_path)\n",
    "\n",
    "        Logger.EVALUATION_LOGGING = evaluation_logging\n",
    "\n",
    "        if Params['OPTIMIZATION_OBJECTIVE'] == 'NAS':\n",
    "            objective_interface = NASInterface()\n",
    "        elif Params['OPTIMIZATION_OBJECTIVE'] == 'Sphere_min':\n",
    "            objective_interface = Sphere(10)\n",
    "        elif Params['OPTIMIZATION_OBJECTIVE'] == 'Sphere_max':\n",
    "            objective_interface = Sphere(10, False)\n",
    "        elif Params['OPTIMIZATION_OBJECTIVE'] == 'Rosenbrock':\n",
    "            objective_interface = Rosenbrock(2)\n",
    "\n",
    "        abc = ArtificialBeeColony(objective_interface)\n",
    "\n",
    "        abc.optimize()\n",
    "\n",
    "        if kill_after:\n",
    "            # Disconnect runtime / free up GPU instance (mainly for Colab)\n",
    "            !kill -9 -1\n",
    "\n",
    "\n",
    "    @staticmethod\n",
    "    def fully_train_topology(config_path=None,\n",
    "                             kill_after=True):\n",
    "        '''Given the current configuration file,\n",
    "        extract the best previously-found topology and fully-train it\n",
    "\n",
    "        Args:\n",
    "            config_path (str, optional): yaml configuration file path; \\\n",
    "            defaults to hard-coded config in :class:`~config.params.Params`\n",
    "        '''\n",
    "\n",
    "        if config_path:\n",
    "            # load yaml config from given path\n",
    "            Params.init_from_yaml(config_path)\n",
    "\n",
    "        Logger.start_log()\n",
    "\n",
    "        # loads architecture and optimizes its weights over a larger number of epochs;\n",
    "        # from_arch sepcifies whether to re-instantiate the network and train from scratch\n",
    "        # or resume training from weights file\n",
    "        res = NASInterface().fully_train_best_model(from_arch=True)\n",
    "\n",
    "        Logger.end_log()\n",
    "\n",
    "        print(res)\n",
    "\n",
    "        if kill_after:\n",
    "            # Disconnect runtime / free up GPU instance\n",
    "            !kill -9 -1\n",
    "\n",
    "\n",
    "# Run HiveNAS\n",
    "HiveNAS.find_topology()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "tjXeDtjA8V6m"
   },
   "source": [
    "### Debug Utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "qPqR0qTviziy"
   },
   "outputs": [],
   "source": [
    "def delete_all():\n",
    "    '''\n",
    "        ***DESTRUCTIVE ACTION***\n",
    "        Removes all weight files and results CSV.\n",
    "        Used for freeing up space from faulty runs.\n",
    "\n",
    "        --Make sure the file paths are entered correctly in Params--\n",
    "    '''\n",
    "    for root, dirs, files in os.walk(Params.get_results_path() + Params['WEIGHT_FILES_SUBPATH']):\n",
    "        for f in files:\n",
    "            filepath = os.path.join(Params.get_results_path() + Params['WEIGHT_FILES_SUBPATH'], f)\n",
    "            os.remove(filepath)\n",
    "\n",
    "    os.remove(os.path.join(Params.get_results_path(), f'{Params[\"CONFIG_VERSION\"]}.csv'))\n",
    "\n",
    "# delete_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "GTGYU9-hzMkC"
   },
   "outputs": [],
   "source": [
    "''' Test ImageAugmentation (before/after plot) '''\n",
    "\n",
    "gc.collect()\n",
    "image_augmentor = ImageDataGenerator(\n",
    "                                    zoom_range = [0.8, 1.1],\n",
    "                                    shear_range= 10,\n",
    "                                    rotation_range=15,\n",
    "                                    width_shift_range=0.1,\n",
    "                                    height_shift_range=0.1,\n",
    "                                    horizontal_flip=True,\n",
    "                                    preprocessing_function=ImgAug.augment,\n",
    "                                    validation_split=0.2)\n",
    "image_augmentor_before = ImageDataGenerator()\n",
    "\n",
    "\n",
    "import requests\n",
    "DownURL = \"https://images.pexels.com/photos/1056251/pexels-photo-1056251.jpeg?crop=entropy&cs=srgb&dl=pexels-ihsan-aditya-1056251.jpg&fit=crop&fm=jpg&h=426&w=640\"\n",
    "img_data = requests.get(DownURL).content\n",
    "\n",
    "FileHandler.create_dir('/content/sample_data/impath/cat')\n",
    "\n",
    "with open('/content/sample_data/impath/cat/cat-small.jpg', 'wb') as handler:\n",
    "    handler.write(img_data)\n",
    "\n",
    "data_before = image_augmentor_before.flow_from_directory(\n",
    "    \"/content/sample_data/impath\",\n",
    "    target_size=(213, 320),\n",
    "    batch_size=1,\n",
    ")\n",
    "data = image_augmentor.flow_from_directory(\n",
    "    \"/content/sample_data/impath\",\n",
    "    target_size=(213, 320),\n",
    "    batch_size=1,\n",
    ")\n",
    "\n",
    "fig, axs = plt.subplots(1,2, figsize=(15,12))\n",
    "\n",
    "plt.subplot(axs[0])\n",
    "plt.imshow(data_before.next()[0][0].astype('int'))\n",
    "plt.title(\"Before\")\n",
    "\n",
    "plt.subplot(axs[1])\n",
    "plt.imshow(data.next()[0][0].astype('int'))\n",
    "plt.title(\"After\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "nZwgEc431p49"
   },
   "outputs": [],
   "source": [
    "''' Unit Testing '''\n",
    "\n",
    "Logger.EVALUATION_LOGGING = False\n",
    "\n",
    "objective_interface = Sphere(10)\n",
    "\n",
    "abc = ArtificialBeeColony(objective_interface)\n",
    "\n",
    "abc.optimize()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "bSvzxJdA2g6t"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "GZwutf5s9vN3"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "provenance": [],
   "toc_visible": true
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}