From 57778611dfbe7497c6ee4f345edf52813dc3e132 Mon Sep 17 00:00:00 2001 From: "Petersen, Brenden Kyle" Date: Wed, 13 Jan 2021 20:38:49 -0800 Subject: [PATCH] ICLR release. --- .gitignore | 13 + LICENSE | 30 + NOTICE | 21 + README.md | 134 ++++ dsr/dsr/__init__.py | 3 + dsr/dsr/baselines/__init__.py | 0 dsr/dsr/baselines/constraints.py | 128 ++++ dsr/dsr/baselines/gpsr.py | 297 ++++++++ dsr/dsr/config.json | 99 +++ dsr/dsr/const.py | 74 ++ dsr/dsr/controller.py | 666 ++++++++++++++++++ dsr/dsr/core.py | 126 ++++ dsr/dsr/cyfunc.pyx | 90 +++ dsr/dsr/functions.py | 195 +++++ dsr/dsr/library.py | 196 ++++++ dsr/dsr/memory.py | 358 ++++++++++ dsr/dsr/prior.py | 527 ++++++++++++++ dsr/dsr/program.py | 640 +++++++++++++++++ dsr/dsr/run.py | 224 ++++++ dsr/dsr/subroutines.py | 120 ++++ dsr/dsr/task/__init__.py | 1 + dsr/dsr/task/regression/__init__.py | 0 dsr/dsr/task/regression/benchmarks.csv | 38 + dsr/dsr/task/regression/dataset.py | 274 +++++++ dsr/dsr/task/regression/function_sets.csv | 13 + dsr/dsr/task/regression/regression.py | 352 +++++++++ dsr/dsr/task/regression/sklearn.py | 35 + dsr/dsr/task/regression/test_sklearn.py | 24 + dsr/dsr/task/task.py | 86 +++ dsr/dsr/test/__init__.py | 0 .../test/data/test_model.data-00000-of-00001 | Bin 0 -> 83444 bytes dsr/dsr/test/data/test_model.index | Bin 0 -> 564 bytes dsr/dsr/test/data/test_model.meta | Bin 0 -> 411999 bytes dsr/dsr/test/generate_test_data.py | 28 + dsr/dsr/test/test_core.py | 47 ++ dsr/dsr/test/test_prior.py | 426 +++++++++++ dsr/dsr/train.py | 508 +++++++++++++ dsr/dsr/utils.py | 154 ++++ dsr/setup.py | 16 + requirements.txt | 12 + 40 files changed, 5955 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 NOTICE create mode 100644 README.md create mode 100644 dsr/dsr/__init__.py create mode 100644 dsr/dsr/baselines/__init__.py create mode 100644 dsr/dsr/baselines/constraints.py create mode 100644 dsr/dsr/baselines/gpsr.py create mode 100644 dsr/dsr/config.json create mode 100644 dsr/dsr/const.py create mode 100644 dsr/dsr/controller.py create mode 100644 dsr/dsr/core.py create mode 100644 dsr/dsr/cyfunc.pyx create mode 100644 dsr/dsr/functions.py create mode 100644 dsr/dsr/library.py create mode 100644 dsr/dsr/memory.py create mode 100644 dsr/dsr/prior.py create mode 100644 dsr/dsr/program.py create mode 100644 dsr/dsr/run.py create mode 100644 dsr/dsr/subroutines.py create mode 100644 dsr/dsr/task/__init__.py create mode 100644 dsr/dsr/task/regression/__init__.py create mode 100644 dsr/dsr/task/regression/benchmarks.csv create mode 100644 dsr/dsr/task/regression/dataset.py create mode 100644 dsr/dsr/task/regression/function_sets.csv create mode 100644 dsr/dsr/task/regression/regression.py create mode 100644 dsr/dsr/task/regression/sklearn.py create mode 100644 dsr/dsr/task/regression/test_sklearn.py create mode 100644 dsr/dsr/task/task.py create mode 100644 dsr/dsr/test/__init__.py create mode 100644 dsr/dsr/test/data/test_model.data-00000-of-00001 create mode 100644 dsr/dsr/test/data/test_model.index create mode 100644 dsr/dsr/test/data/test_model.meta create mode 100644 dsr/dsr/test/generate_test_data.py create mode 100644 dsr/dsr/test/test_core.py create mode 100644 dsr/dsr/test/test_prior.py create mode 100644 dsr/dsr/train.py create mode 100644 dsr/dsr/utils.py create mode 100644 dsr/setup.py create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..09ce2199 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +*.DS_Store +*.pyc +*.egg* +venv* +dsr/dsr/summary* +*log_* +.gitignore +.ipynb_checkpoints +~$* +*.vscode/ +dsr/build +dsr/dsr/cyfunc* +**/log/ \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..92216caa --- /dev/null +++ b/LICENSE @@ -0,0 +1,30 @@ +BSD 3-Clause License + +Copyright (c) 2018, Lawrence Livermore National Security, LLC +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/NOTICE b/NOTICE new file mode 100644 index 00000000..3737d5a8 --- /dev/null +++ b/NOTICE @@ -0,0 +1,21 @@ +This work was produced under the auspices of the U.S. Department of +Energy by Lawrence Livermore National Laboratory under Contract +DE-AC52-07NA27344. + +This work was prepared as an account of work sponsored by an agency of +the United States Government. Neither the United States Government nor +Lawrence Livermore National Security, LLC, nor any of their employees +makes any warranty, expressed or implied, or assumes any legal liability +or responsibility for the accuracy, completeness, or usefulness of any +information, apparatus, product, or process disclosed, or represents that +its use would not infringe privately owned rights. + +Reference herein to any specific commercial product, process, or service +by trade name, trademark, manufacturer, or otherwise does not necessarily +constitute or imply its endorsement, recommendation, or favoring by the +United States Government or Lawrence Livermore National Security, LLC. + +The views and opinions of authors expressed herein do not necessarily +state or reflect those of the United States Government or Lawrence +Livermore National Security, LLC, and shall not be used for advertising +or product endorsement purposes. diff --git a/README.md b/README.md new file mode 100644 index 00000000..61248c9b --- /dev/null +++ b/README.md @@ -0,0 +1,134 @@ +# Deep symbolic regression + +Deep symbolic regression (DSR) is a deep learning algorithm for symbolic regression--the task of recovering tractable mathematical expressions from an input dataset. The package `dsr` contains the code for DSR, including a single-point, parallelized launch script (`dsr/run.py`), baseline genetic programming-based symbolic regression algorithm, and an sklearn-like interface for use with your own data. + +This code supports the ICLR 2021 paper [Deep symbolic regression: Recovering mathematical expressions from data via risk-seeking policy gradients](https://openreview.net/forum?id=m5Qsh0kBQG). + +# Installation + +Installation is straightforward in a Python 3 virtual environment using Pip. From the repository root: + +``` +python3 -m venv venv3 # Create a Python 3 virtual environment +source venv3/bin/activate # Activate the virtual environmnet +pip install -r requirements.txt # Install Python dependencies +export CFLAGS="-I $(python -c "import numpy; print(numpy.get_include())") $CFLAGS" # Needed on Mac to prevent fatal error: 'numpy/arrayobject.h' file not found +pip install -e ./dsr # Install DSR package +``` + +To perform experiments involving the GP baseline, you will need the additional package `deap`. + +# Example usage + +To try out DSR, use the following command from the repository root: + +``` +python -m dsr.run ./dsr/dsr/config.json --b=Nguyen-6 +``` + +This should solve in around 50 training steps (~30 seconds on a laptop). + +# Getting started + +## Configuring runs + +DSR uses JSON files to configure training. + +Top-level key "task" specifies details of the benchmark expression for DSR or GP. See docs in `regression.py` for details. + +Top-level key "training" specifies the training hyperparameters for DSR. See docs in `train.py` for details. + +Top-level key "controller" specifies the RNN controller hyperparameters for DSR. See docs for in `controller.py` for details. + +Top-level key "gp" specifies the hyperparameters for GP if using the GP baseline. See docs for `dsr.baselines.gspr.GP` for details. + +## Launching runs + +After configuring a run, launching it is simple: + +``` +python -m dsr.run [PATH_TO_CONFIG] [--OPTIONS] +``` + +## Sklearn interface + +DSR also provides an [sklearn-like regressor interface](https://scikit-learn.org/stable/modules/generated/sklearn.base.RegressorMixin.html). Example usage: + +``` +from dsr import DeepSymbolicRegressor +import numpy as np + +# Generate some data +np.random.seed(0) +X = np.random.random((10, 2)) +y = np.sin(X[:,0]) + X[:,1] ** 2 + +# Create the model +model = DeepSymbolicRegressor("config.json") + +# Fit the model +model.fit(X, y) # Should solve in ~10 seconds + +# View the best expression +print(model.program_.pretty()) + +# Make predictions +model.predict(2 * X) +``` + +## Using an external dataset + +To use your own dataset, simply provide the path to the `"dataset"` key in the config, and give your task an arbitary name. + +``` +"task": { + "task_type": "regression", + "name": "my_task", + "dataset": "./path/to/my_dataset.csv", + ... +} +``` + +Then run DSR: + +``` +python -m dsr.run path/to/config.json +``` + +Note the `--b` flag matches the name of the CSV file (-`.csv` ). + +## Command-line examples + +Show command-line help and quit + +``` +python -m dsr.run --help +``` + +Train 2 indepdent runs of DSR on the Nguyen-1 benchmark using 2 cores + +``` +python -m dsr.run config.json --b=Nguyen-1 --mc=2 --num_cores=2 +``` + +Train DSR on all 12 Nguyen benchmarks using 12 cores + +``` +python -m dsr.run config.json --b=Nguyen --num_cores=12 +``` + +Train 2 independent runs of GP on Nguyen-1 + +``` +python -m dsr.run config.json --method=gp --b=Nguyen-1 --mc=2 --num_cores=2 +``` + +Train DSR on Nguyen-1 and Nguyen-4 + +``` +python -m dsr.run config.json --b=Nguyen-1 --b=Nguyen-4 +``` + +# Release + +LLNL-CODE-647188 diff --git a/dsr/dsr/__init__.py b/dsr/dsr/__init__.py new file mode 100644 index 00000000..b18aa77a --- /dev/null +++ b/dsr/dsr/__init__.py @@ -0,0 +1,3 @@ +from dsr.core import DeepSymbolicOptimizer +from dsr.task.regression.sklearn import DeepSymbolicRegressor + diff --git a/dsr/dsr/baselines/__init__.py b/dsr/dsr/baselines/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dsr/dsr/baselines/constraints.py b/dsr/dsr/baselines/constraints.py new file mode 100644 index 00000000..45c4526e --- /dev/null +++ b/dsr/dsr/baselines/constraints.py @@ -0,0 +1,128 @@ +"""Defines constraints for GP individuals, to be used as decorators for +evolutionary operations.""" + +from dsr.functions import UNARY_TOKENS, BINARY_TOKENS + +TRIG_TOKENS = ["sin", "cos", "tan", "csc", "sec", "cot"] + +# Define inverse tokens +INVERSE_TOKENS = { + "exp" : "log", + "neg" : "neg", + "inv" : "inv", + "sqrt" : "n2" +} + +# Add inverse trig functions +INVERSE_TOKENS.update({ + t : "arc" + t for t in TRIG_TOKENS + }) + +# Add reverse +INVERSE_TOKENS.update({ + v : k for k, v in INVERSE_TOKENS.items() + }) + +DEBUG = False + + +def check_inv(ind): + """Returns True if two sequential tokens are inverse unary operators.""" + + names = [node.name for node in ind] + for i, name in enumerate(names[:-1]): + if name in INVERSE_TOKENS and names[i+1] == INVERSE_TOKENS[name]: + if DEBUG: + print("Constrained inverse:", ind) + return True + return False + + +def check_const(ind): + """Returns True if children of a parent are all const tokens.""" + + names = [node.name for node in ind] + for i, name in enumerate(names): + if name in UNARY_TOKENS and names[i+1] == "const": + if DEBUG: + print("Constrained const (unary)", ind) + return True + if name in BINARY_TOKENS and names[i+1] == "const" and names[i+1] == "const": + if DEBUG: + print(print("Constrained const (binary)", ind)) + return True + return False + + +def check_trig(ind): + """Returns True if a descendant of a trig operator is another trig + operator.""" + + names = [node.name for node in ind] + trig_descendant = False # True when current node is a descendant of a trig operator + trig_dangling = None # Number of unselected nodes in trig subtree + for i, name in enumerate(names): + if name in TRIG_TOKENS: + if trig_descendant: + if DEBUG: + print("Constrained trig:", ind) + return True + trig_descendant = True + trig_dangling = 1 + elif trig_descendant: + if name in BINARY_TOKENS: + trig_dangling += 1 + elif name not in UNARY_TOKENS: + trig_dangling -= 1 + if trig_dangling == 0: + trig_descendant = False + return False + + +def make_check_min_len(min_length): + """Creates closure for minimum length constraint""" + + def check_min_len(ind): + """Returns True if individual is less than minimum length""" + + if len(ind) < min_length: + if DEBUG: + print("Constrained min len: {} (length {})".format(ind, len(ind))) + return True + + return False + + return check_min_len + + +def make_check_max_len(max_length): + """Creates closure for maximum length constraint""" + + def check_max_len(ind): + """Returns True if individual is greater than maximum length""" + + if len(ind) > max_length: + if DEBUG: + print("Constrained max len: {} (length {})".format(ind, len(ind))) + return True + + return False + + return check_max_len + + +def make_check_num_const(max_const): + """Creates closure for maximum number of constants constraint""" + + def check_num_const(ind): + """Returns True if individual has more than max_const const tokens""" + + num_const = len([t for t in ind if t.name == "const"]) + if num_const > max_const: + if DEBUG: + print("Constrained max const: {} ({} consts)".format(ind, num_const)) + return True + + return False + + return check_num_const diff --git a/dsr/dsr/baselines/gpsr.py b/dsr/dsr/baselines/gpsr.py new file mode 100644 index 00000000..f3e7c186 --- /dev/null +++ b/dsr/dsr/baselines/gpsr.py @@ -0,0 +1,297 @@ +import random +import operator +import importlib +from functools import partial + +import numpy as np + +from dsr.functions import function_map +from dsr.const import make_const_optimizer + +from . import constraints + + +GP_MOD = "deap" +OBJECTS = ["base", "gp", "creator", "tools", "algorithms"] +gp = importlib.import_module(GP_MOD + ".gp") +base = importlib.import_module(GP_MOD + ".base") +creator = importlib.import_module(GP_MOD + ".creator") +tools = importlib.import_module(GP_MOD + ".tools") +algorithms = importlib.import_module(GP_MOD + ".algorithms") + + +class GP(): + """Genetic-programming based symbolic regression class""" + + def __init__(self, dataset, metric="nmse", population_size=1000, + generations=1000, n_samples=None, tournament_size=3, + p_crossover=0.5, p_mutate=0.1, + const_range=[-1, 1], const_optimizer="scipy", + const_params=None, seed=0, early_stopping=False, + threshold=1e-12, verbose=True, protected=True, + pareto_front=False, + # Constraint hyperparameters + constrain_const=True, + constrain_trig=True, + constrain_inv=True, + constrain_min_len=True, + constrain_max_len=True, + constrain_num_const=True, + min_length=4, + max_length=30, + max_const=3): + + self.dataset = dataset + self.fitted = False + + assert n_samples is None or generations is None, "At least one of 'n_samples' or 'generations' must be None." + if generations is None: + generations = int(n_samples / population_size) + + # Set hyperparameters + self.population_size = population_size + self.generations = generations + self.tournament_size = tournament_size + self.p_mutate = p_mutate + self.p_crossover = p_crossover + self.seed = seed + self.early_stopping = early_stopping + self.threshold = threshold + self.verbose = verbose + self.pareto_front = pareto_front + + # Fitness function used during training + # Includes closure for fitness function metric and training data + fitness = partial(self.make_fitness(metric), y=dataset.y_train, var_y=np.var(dataset.y_train)) # Function of y_hat + self.fitness = partial(self.compute_fitness, optimize=True, fitness=fitness, X=dataset.X_train.T) # Function of individual + + # Test NMSE, used as final performance metric + # Includes closure for test data + nmse_test = partial(self.make_fitness("nmse"), y=dataset.y_test, var_y=np.var(dataset.y_test)) # Function of y_hat + self.nmse_test = partial(self.compute_fitness, optimize=False, fitness=nmse_test, X=dataset.X_test.T) # Function of individual + + # Noiseless test NMSE, only used to determine success for final performance + # Includes closure for noiseless test data + nmse_test_noiseless = partial(self.make_fitness("nmse"), y=dataset.y_test_noiseless, var_y=np.var(dataset.y_test_noiseless)) # Function of y_hat + self.nmse_test_noiseless = partial(self.compute_fitness, optimize=False, fitness=nmse_test_noiseless, X=dataset.X_test.T) # Function of individual + self.success = lambda ind : self.nmse_test_noiseless(ind)[0] < self.threshold # Function of individual + + # Create the primitive set + pset = gp.PrimitiveSet("MAIN", dataset.X_train.shape[1]) + + # Add input variables + rename_kwargs = {"ARG{}".format(i) : "x{}".format(i + 1) for i in range(dataset.n_input_var)} + pset.renameArguments(**rename_kwargs) + + # Add primitives + for op_name in dataset.function_set: + if op_name == "const": + continue + assert op_name in function_map, "Operation {} not recognized.".format(op_name) + + # Prepend available protected operators with "protected_" + if protected and not op_name.startswith("protected_"): + protected_op_name = "protected_{}".format(op_name) + if protected_op_name in function_map: + op_name = protected_op_name + + op = function_map[op_name] + pset.addPrimitive(op.function, op.arity, name=op.name) + + # # Add constant + # if "const" in dataset.function_set: + # pset.addEphemeralConstant("const", lambda : random.uniform(const_range[0], const_range[1])) + + # Add constant + const = "const" in dataset.function_set + if const: + const_params = const_params if const_params is not None else {} + self.const_opt = make_const_optimizer(const_optimizer, **const_params) + pset.addTerminal(1.0, name="const") + + # Create custom fitness and individual classes + if self.pareto_front: + # Fitness it compared lexographically, so second dimension + # (complexity) is only used in selection if first dimension (error) + # is the same. + creator.create("FitnessMin", base.Fitness, weights=(-1.0, -1.0)) + else: + creator.create("FitnessMin", base.Fitness, weights=(-1.0,)) + creator.create("Individual", gp.PrimitiveTree, fitness=creator.FitnessMin) + + # Define the evolutionary operators + self.toolbox = base.Toolbox() + self.toolbox.register("expr", gp.genHalfAndHalf, pset=pset, min_=1, max_=2) + self.toolbox.register("individual", tools.initIterate, creator.Individual, self.toolbox.expr) + self.toolbox.register("population", tools.initRepeat, list, self.toolbox.individual) + self.toolbox.register("compile", gp.compile, pset=pset) + self.toolbox.register("evaluate", self.fitness) + self.toolbox.register("select", tools.selTournament, tournsize=tournament_size) + self.toolbox.register("mate", gp.cxOnePoint) + self.toolbox.register("expr_mut", gp.genFull, min_=0, max_=2) + self.toolbox.register('mutate', gp.mutUniform, expr=self.toolbox.expr_mut, pset=pset) + + # Define constraints, each defined by a func : gp.Individual -> bool. + # We decorate mutation/crossover operators with constrain, which + # replaces a child with a random parent if func(ind) is True. + constrain = partial(gp.staticLimit, max_value=0) # Constraint decorator + funcs = [] + if constrain_min_len: + funcs.append(constraints.make_check_min_len(min_length)) # Minimum length + if constrain_max_len: + funcs.append(constraints.make_check_max_len(max_length)) # Maximum length + if constrain_inv: + funcs.append(constraints.check_inv) # Subsequence inverse unary operators + if constrain_trig: + funcs.append(constraints.check_trig) # Nested trig operators + if constrain_const and const: + funcs.append(constraints.check_const) # All children are constants + if constrain_num_const and const: + funcs.append(constraints.make_check_num_const(max_const)) # Number of constants + for func in funcs: + for variation in ["mate", "mutate"]: + self.toolbox.decorate(variation, constrain(func)) + + # Create the training function + self.algorithm = algorithms.eaSimple + + + def compute_fitness(self, individual, fitness, X, optimize=False): + """Compute the given fitness function on an individual using X.""" + + if optimize: + # Retrieve symbolic constants + const_idxs = [i for i, node in enumerate(individual) if node.name == "const"] + + # Check if best individual (or any individual in Pareto front) has success=True + # (i.e. NMSE below threshold on noiseless test set) + if self.early_stopping and any([self.success(ind) for ind in self.hof]): + return (999,) + + if optimize and len(const_idxs) > 0: + + # Objective function for evaluating constants + def obj(consts): + for i, const in zip(const_idxs, consts): + individual[i] = gp.Terminal(const, False, object) + individual[i].name = "const" # For good measure + f = self.toolbox.compile(expr=individual) + y_hat = f(*X) + y = self.dataset.y_train + if np.isfinite(y_hat).all(): + # Squash error to prevent consts from becoming inf + return -1/(1 + np.mean((y - y_hat)**2)) + else: + return 0 + + # Do the optimization and set the optimized constants + x0 = np.ones(len(const_idxs)) + optimized_consts = self.const_opt(obj, x0) + for i, const in zip(const_idxs, optimized_consts): + individual[i] = gp.Terminal(const, False, object) + individual[i].name = "const" # This is necessary to ensure the constant is re-optimized in the next generation + + # Execute the individual + f = self.toolbox.compile(expr=individual) + with np.errstate(all="ignore"): + y_hat = f(*X) + + # Check for validity + if np.isfinite(y_hat).all(): + fitness = (fitness(y_hat=y_hat),) + else: + fitness = (np.inf,) + + # Compute complexity (only if using Pareto front) + if self.pareto_front: + complexity = sum([function_map[prim.name].complexity \ + if prim.name in function_map \ + else 1 for prim in individual]) + fitness += (complexity,) + + return fitness + + + def train(self): + """Train the GP""" + + if self.fitted: + raise RuntimeError("This GP has already been fitted!") + + random.seed(self.seed) + + pop = self.toolbox.population(n=self.population_size) + if self.pareto_front: + self.hof = tools.ParetoFront() + else: + self.hof = tools.HallOfFame(maxsize=1) + + stats_fit = tools.Statistics(lambda p : p.fitness.values[0]) + stats_fit.register("avg", np.mean) + stats_fit.register("min", np.min) + stats_size = tools.Statistics(len) + stats_size.register("avg", np.mean) + mstats = tools.MultiStatistics(fitness=stats_fit, size=stats_size) + + pop, logbook = self.algorithm(population=pop, + toolbox=self.toolbox, + cxpb=self.p_crossover, + mutpb=self.p_mutate, + ngen=self.generations, + stats=mstats, + halloffame=self.hof, + verbose=self.verbose) + + self.fitted = True + + # Delete custom classes + del creator.FitnessMin + del creator.Individual + if "const" in dir(gp): + del gp.const + + # The best individual is the first one in self.hof with success=True, + # otherwise the highest reward. This mimics DSR's train.py. + ind_best = None + for ind in self.hof: + if self.success(ind): + ind_best = ind + break + ind_best = ind_best if ind_best is not None else self.hof[0] # first element in self.hof is the fittest + + if self.verbose: + print("Printing {}:".format("Pareto front" if self.pareto_front else "hall of fame")) + print("Fitness | Individual") + for ind in self.hof: + print(ind.fitness, [token.name for token in ind]) + + return ind_best, logbook + + + def make_fitness(self, metric): + """Generates a fitness function by name""" + + if metric == "mse": + fitness = lambda y, y_hat, var_y : np.mean((y - y_hat)**2) + + elif metric == "rmse": + fitness = lambda y, y_hat, var_y : np.sqrt(np.mean((y - y_hat)**2)) + + elif metric == "nmse": + fitness = lambda y, y_hat, var_y : np.mean((y - y_hat)**2 / var_y) + + elif metric == "nrmse": + fitness = lambda y, y_hat, var_y : np.sqrt(np.mean((y - y_hat)**2 / var_y)) + + # Complementary inverse NMSE + elif metric == "cinv_nmse": + fitness = lambda y, y_hat, var_y : 1 - 1/(1 + np.mean((y - y_hat)**2 / var_y)) + + # Complementary inverse NRMSE + elif metric == "cinv_nrmse": + fitness = lambda y, y_hat, var_y : 1 - 1/(1 + np.sqrt(np.mean((y - y_hat)**2 / var_y))) + + else: + raise ValueError("Metric not recognized.") + + return fitness diff --git a/dsr/dsr/config.json b/dsr/dsr/config.json new file mode 100644 index 00000000..831c8707 --- /dev/null +++ b/dsr/dsr/config.json @@ -0,0 +1,99 @@ +{ + "task": { + "task_type" : "regression", + "name" : "Nguyen-1", + "function_set": null, + "dataset" : { + "name" : null, + "noise": null, + "dataset_size_multiplier": 1.0 + }, + "metric" : "inv_nrmse", + "metric_params" : [1.0], + "threshold" : 1e-12, + "protected" : false, + "reward_noise" : 0.0 + }, + "prior": { + "length" : {"min_" : 4, "max_" : 30}, + "repeat" : {"tokens" : "const", "max_" : 3}, + "inverse" : {}, + "trig" : {}, + "const" : {} + }, + "training": { + "logdir": "./log", + "n_epochs": null, + "n_samples": 2000000, + "batch_size": 1000, + "complexity": "length", + "complexity_weight": 0.0, + "const_optimizer": "scipy", + "const_params": {}, + "alpha": 0.5, + "epsilon": 0.05, + "verbose": true, + "baseline": "R_e", + "b_jumpstart": false, + "n_cores_batch": 1, + "summary": false, + "debug": 0, + "output_file": null, + "save_all_r": false, + "early_stopping": true, + "pareto_front": false, + "hof": 100 + }, + "controller": { + "cell": "lstm", + "num_layers": 1, + "num_units": 32, + "initializer": "zeros", + "embedding": false, + "embedding_size": 8, + "optimizer": "adam", + "learning_rate": 0.0005, + "observe_action": false, + "observe_parent": true, + "observe_sibling": true, + "entropy_weight": 0.005, + "ppo": false, + "ppo_clip_ratio": 0.2, + "ppo_n_iters": 10, + "ppo_n_mb": 4, + "pqt": false, + "pqt_k": 10, + "pqt_batch_size": 1, + "pqt_weight": 200.0, + "pqt_use_pg": false, + "max_length": 30 + }, + "gp": { + "population_size": 1000, + "generations": null, + "n_samples" : 2000000, + "tournament_size": 2, + "metric": "nmse", + "const_range": [ + -1.0, + 1.0 + ], + "p_crossover": 0.95, + "p_mutate": 0.03, + "seed": 0, + "early_stopping": true, + "pareto_front": false, + "threshold": 1e-12, + "verbose": false, + "protected": true, + "constrain_const": true, + "constrain_trig": true, + "constrain_inv": true, + "constrain_min_len": true, + "constrain_max_len": true, + "constrain_num_const": true, + "min_length": 4, + "max_length": 30, + "max_const" : 3 + } +} diff --git a/dsr/dsr/const.py b/dsr/dsr/const.py new file mode 100644 index 00000000..dd41cbf9 --- /dev/null +++ b/dsr/dsr/const.py @@ -0,0 +1,74 @@ +"""Constant optimizer used for deep symbolic regression.""" + +from functools import partial + +import numpy as np +from scipy.optimize import minimize + + +def make_const_optimizer(name, **kwargs): + """Returns a ConstOptimizer given a name and keyword arguments""" + + const_optimizers = { + None : Dummy, + "dummy" : Dummy, + "scipy" : ScipyMinimize, + } + + return const_optimizers[name](**kwargs) + + +class ConstOptimizer(object): + """Base class for constant optimizer""" + + def __init__(self, **kwargs): + self.kwargs = kwargs + + + def __call__(self, f, x0): + """ + Optimizes an objective function from an initial guess. + + The objective function is the negative of the base reward (reward + without penalty) used for training. Optimization excludes any penalties + because they are constant w.r.t. to the constants being optimized. + + Parameters + ---------- + f : function mapping np.ndarray to float + Objective function (negative base reward). + + x0 : np.ndarray + Initial guess for constant placeholders. + + Returns + ------- + x : np.ndarray + Vector of optimized constants. + """ + raise NotImplementedError + + +class Dummy(ConstOptimizer): + """Dummy class that selects the initial guess for each constant""" + + def __init__(self, **kwargs): + super(Dummy, self).__init__(**kwargs) + + + def __call__(self, f, x0): + return x0 + + +class ScipyMinimize(ConstOptimizer): + """SciPy's non-linear optimizer""" + + def __init__(self, **kwargs): + super(ScipyMinimize, self).__init__(**kwargs) + + + def __call__(self, f, x0): + with np.errstate(divide='ignore'): + opt_result = partial(minimize, **self.kwargs)(f, x0) + x = opt_result['x'] + return x diff --git a/dsr/dsr/controller.py b/dsr/dsr/controller.py new file mode 100644 index 00000000..1872c7f0 --- /dev/null +++ b/dsr/dsr/controller.py @@ -0,0 +1,666 @@ +"""Controller used to generate distribution over hierarchical, variable-length objects.""" + +import tensorflow as tf +import numpy as np + +from dsr.program import Program +from dsr.memory import Batch +from dsr.subroutines import parents_siblings +from dsr.prior import LengthConstraint + + +class LinearWrapper(tf.contrib.rnn.LayerRNNCell): + """ + RNNCell wrapper that adds a linear layer to the output. + + See: https://github.com/tensorflow/models/blob/master/research/brain_coder/single_task/pg_agent.py + """ + + def __init__(self, cell, output_size): + self.cell = cell + self._output_size = output_size + + def __call__(self, inputs, state, scope=None): + with tf.variable_scope(type(self).__name__): + outputs, state = self.cell(inputs, state, scope=scope) + logits = tf.layers.dense(outputs, units=self._output_size) + + return logits, state + + @property + def output_size(self): + return self._output_size + + @property + def state_size(self): + return self.cell.state_size + + def zero_state(self, batch_size, dtype): + return self.cell.zero_state(batch_size, dtype) + + +class Controller(object): + """ + Recurrent neural network (RNN) controller used to generate expressions. + + Specifically, the RNN outputs a distribution over pre-order traversals of + symbolic expression trees. It is trained using REINFORCE with baseline. + + Parameters + ---------- + sess : tf.Session + TenorFlow Session object. + + prior : dsr.prior.JointPrior + JointPrior object used to adjust probabilities during sampling. + + summary : bool + Write tensorboard summaries? + + debug : int + Debug level, also used in learn(). 0: No debug. 1: Print shapes and + number of parameters for each variable. + + cell : str + Recurrent cell to use. Supports 'lstm' and 'gru'. + + num_layers : int + Number of RNN layers. + + num_units : int or list of ints + Number of RNN cell units in each of the RNN's layers. If int, the value + is repeated for each layer. + + initiailizer : str + Initializer for the recurrent cell. Supports 'zeros' and 'var_scale'. + + embedding : bool + Embed each observation? + + embedding_size : int + Size of embedding for each observation if embedding=True. + + optimizer : str + Optimizer to use. Supports 'adam', 'rmsprop', and 'sgd'. + + learning_rate : float + Learning rate for optimizer. + + observe_action : bool + Observe previous action token? + + observe_parent : bool + Observe parent token? + + observe_sibling : bool + Observe sibling token? + + entropy_weight : float + Coefficient for entropy bonus. + + ppo : bool + Use proximal policy optimization (instead of vanilla policy gradient)? + + ppo_clip_ratio : float + Clip ratio to use for PPO. + + ppo_n_iters : int + Number of optimization iterations for PPO. + + ppo_n_mb : int + Number of minibatches per optimization iteration for PPO. + + pqt : bool + Train with priority queue training (PQT)? + + pqt_k : int + Size of priority queue. + + pqt_batch_size : int + Size of batch to sample (with replacement) from priority queue. + + pqt_weight : float + Coefficient for PQT loss function. + + pqt_use_pg : bool + Use policy gradient loss when using PQT? + + max_length : int or None + Maximum sequence length. This will be overridden if a LengthConstraint + with a maximum length is part of the prior. + + """ + + def __init__(self, sess, prior, debug=0, summary=True, + # RNN cell hyperparameters + cell='lstm', + num_layers=1, + num_units=32, + initializer='zeros', + # Embedding hyperparameters + embedding=False, + embedding_size=4, + # Optimizer hyperparameters + optimizer='adam', + learning_rate=0.001, + # Observation space hyperparameters + observe_action=True, + observe_parent=True, + observe_sibling=True, + # Loss hyperparameters + entropy_weight=0.0, + # PPO hyperparameters + ppo=False, + ppo_clip_ratio=0.2, + ppo_n_iters=10, + ppo_n_mb=4, + # PQT hyperparameters + pqt=False, + pqt_k=10, + pqt_batch_size=1, + pqt_weight=200.0, + pqt_use_pg=False, + # Other hyperparameters + max_length=None): + + self.sess = sess + self.prior = prior + self.summary = summary + self.rng = np.random.RandomState(0) # Used for PPO minibatch sampling + + lib = Program.library + + # Find max_length from the LengthConstraint prior, if it exists + prior_max_length = None + for single_prior in self.prior.priors: + if isinstance(single_prior, LengthConstraint): + if single_prior.max is not None: + prior_max_length = single_prior.max + self.max_length = prior_max_length + break + if prior_max_length is None: + assert max_length is not None, "max_length must be specified if "\ + "there is no LengthConstraint." + self.max_length = max_length + print("WARNING: Maximum length not constrained. Sequences will " + "stop at {} and complete by repeating the first input " + "variable.".format(self.max_length)) + elif max_length is not None and max_length != self.max_length: + print("WARNING: max_length ({}) will be overridden by value from " + "LengthConstraint ({}).".format(max_length, self.max_length)) + max_length = self.max_length + + # Hyperparameters + self.observe_parent = observe_parent + self.observe_sibling = observe_sibling + self.entropy_weight = entropy_weight + self.ppo = ppo + self.ppo_n_iters = ppo_n_iters + self.ppo_n_mb = ppo_n_mb + self.pqt = pqt + self.pqt_k = pqt_k + self.pqt_batch_size = pqt_batch_size + + n_choices = lib.L + + # Placeholders, computed after instantiating expressions + self.batch_size = tf.placeholder(dtype=tf.int32, shape=(), name="batch_size") + self.baseline = tf.placeholder(dtype=tf.float32, shape=(), name="baseline") + + # Parameter assertions/warnings + assert observe_action + observe_parent + observe_sibling > 0, "Must include at least one observation." + + self.compute_parents_siblings = any([self.observe_parent, + self.observe_sibling, + self.prior.requires_parents_siblings]) + + # Build controller RNN + with tf.name_scope("controller"): + + def make_initializer(name): + if name == "zeros": + return tf.zeros_initializer() + if name == "var_scale": + return tf.contrib.layers.variance_scaling_initializer( + factor=0.5, mode='FAN_AVG', uniform=True, seed=0) + raise ValueError("Did not recognize initializer '{}'".format(name)) + + def make_cell(name, num_units, initializer): + if name == 'lstm': + return tf.nn.rnn_cell.LSTMCell(num_units, initializer=initializer) + if name == 'gru': + return tf.nn.rnn_cell.GRUCell(num_units, kernel_initializer=initializer, bias_initializer=initializer) + raise ValueError("Did not recognize cell type '{}'".format(name)) + + # Create recurrent cell + if isinstance(num_units, int): + num_units = [num_units] * num_layers + initializer = make_initializer(initializer) + cell = tf.contrib.rnn.MultiRNNCell( + [make_cell(cell, n, initializer=initializer) for n in num_units]) + cell = LinearWrapper(cell=cell, output_size=n_choices) + + # Define input dimensions + n_action_inputs = n_choices + 1 # lib tokens + empty token + n_parent_inputs = n_choices + 1 - len(lib.terminal_tokens) # Parent sub-lib tokens + empty token + n_sibling_inputs = n_choices + 1 # lib tokens + empty tokens + + # Create embeddings + if embedding: + with tf.variable_scope("embeddings", + initializer=tf.random_uniform_initializer(minval=-1.0, maxval=1.0, seed=0)): + if observe_action: + action_embeddings = tf.get_variable("action_embeddings", [n_action_inputs, embedding_size], trainable=True) + if observe_parent: + parent_embeddings = tf.get_variable("parent_embeddings", [n_parent_inputs, embedding_size], trainable=True) + if observe_sibling: + sibling_embeddings = tf.get_variable("sibling_embeddings", [n_sibling_inputs, embedding_size], trainable=True) + + # First observation is all empty tokens + initial_obs = tuple() + for n in [n_action_inputs, n_parent_inputs, n_sibling_inputs]: + obs = tf.constant(n - 1, dtype=np.int32) + obs = tf.broadcast_to(obs, [self.batch_size]) + initial_obs += (obs,) + + # Get initial prior + initial_prior = self.prior.initial_prior() + initial_prior = tf.constant(initial_prior, dtype=tf.float32) + prior_dims = tf.stack([self.batch_size, n_choices]) + initial_prior = tf.broadcast_to(initial_prior, prior_dims) + # arities = np.array([Program.arities[i] for i in range(n_choices)]) + # prior = np.zeros(n_choices, dtype=np.float32) + # if self.min_length is not None and self.min_length > 1: + # prior[arities == 0] = -np.inf + # prior = tf.constant(prior, dtype=tf.float32) + # prior_dims = tf.stack([self.batch_size, n_choices]) + # prior = tf.broadcast_to(prior, prior_dims) + # initial_prior = prior + + + # Returns concatenated one-hot or embeddings from observation tokens + # Used for both raw_rnn and dynamic_rnn + def get_input(obs): + action, parent, sibling = obs + observations = [] + if observe_action: + if embedding: + obs = tf.nn.embedding_lookup(action_embeddings, action) + else: + obs = tf.one_hot(action, depth=n_action_inputs) + observations.append(obs) + if observe_parent: + if embedding: + obs = tf.nn.embedding_lookup(parent_embeddings, parent) + else: + obs = tf.one_hot(parent, depth=n_parent_inputs) + observations.append(obs) + if observe_sibling: + if embedding: + obs = tf.nn.embedding_lookup(sibling_embeddings, sibling) + else: + obs = tf.one_hot(sibling, depth=n_sibling_inputs) + observations.append(obs) + input_ = tf.concat(observations, -1) + return input_ + + + # Applies constraints + def get_action_parent_sibling_prior_dangling(actions, dangling): + n = actions.shape[0] # Batch size + i = actions.shape[1] - 1 # Current index + action = actions[:, -1] # Current action + + # Depending on the constraints, may need to compute parents and siblings + if self.compute_parents_siblings: + parent, sibling = parents_siblings(actions, arities=lib.arities, parent_adjust=lib.parent_adjust) + else: + parent = np.zeros(n, dtype=np.int32) + sibling = np.zeros(n, dtype=np.int32) + + # Update dangling with (arity - 1) for each element in action + dangling += lib.arities[action] - 1 + + prior = self.prior(actions, parent, sibling, dangling) + + return action, parent, sibling, prior, dangling + + + # Given the actions chosen so far, return the observation, the prior, and the updated dangling + # Uses py_func to retrieve action/parent/sibling/dangling + def get_next_obs_prior_dangling(actions_ta, dangling): + + # Get current action batch + actions = tf.transpose(actions_ta.stack()) # Shape: (?, time) + + # Compute parent, sibling, prior, and dangling + action, parent, sibling, prior, dangling = tf.py_func(func=get_action_parent_sibling_prior_dangling, + inp=[actions, dangling], + Tout=[tf.int32, tf.int32, tf.int32, tf.float32, tf.int32]) + + # Observe previous action, parent, and/or sibling + obs = (action, parent, sibling) + + # Set the shapes for returned Tensors + action.set_shape([None]) + parent.set_shape([None]) + sibling.set_shape([None]) + prior.set_shape([None, lib.L]) + dangling.set_shape([None]) + + return obs, prior, dangling + + + # Define loop function to be used by tf.nn.raw_rnn. + initial_cell_input = get_input(initial_obs) + def loop_fn(time, cell_output, cell_state, loop_state): + + if cell_output is None: # time == 0 + finished = tf.zeros(shape=[self.batch_size], dtype=tf.bool) + obs = initial_obs + next_input = get_input(obs) + next_cell_state = cell.zero_state(batch_size=self.batch_size, dtype=tf.float32) # 2-tuple, each shape (?, num_units) + emit_output = None + actions_ta = tf.TensorArray(dtype=tf.int32, size=0, dynamic_size=True, clear_after_read=False) # Read twice + obs_tas = (tf.TensorArray(dtype=tf.int32, size=0, dynamic_size=True, clear_after_read=True), # Action inputs + tf.TensorArray(dtype=tf.int32, size=0, dynamic_size=True, clear_after_read=True), # Parent inputs + tf.TensorArray(dtype=tf.int32, size=0, dynamic_size=True, clear_after_read=True)) # Sibling inputs + priors_ta = tf.TensorArray(dtype=tf.float32, size=0, dynamic_size=True, clear_after_read=True) + prior = initial_prior + lengths = tf.ones(shape=[self.batch_size], dtype=tf.int32) + dangling = tf.ones(shape=[self.batch_size], dtype=tf.int32) + next_loop_state = ( + actions_ta, + obs_tas, + priors_ta, + obs, + prior, + dangling, + lengths, # Unused until implementing variable length + finished) + else: + actions_ta, obs_tas, priors_ta, obs, prior, dangling, lengths, finished = loop_state + logits = cell_output + prior + next_cell_state = cell_state + emit_output = logits + action = tf.multinomial(logits=logits, num_samples=1, output_dtype=tf.int32, seed=1)[:, 0] + # When implementing variable length: + # action = tf.where( + # tf.logical_not(finished), + # tf.multinomial(logits=logits, num_samples=1, output_dtype=tf.int32)[:, 0], + # tf.zeros(shape=[self.batch_size], dtype=tf.int32)) + next_actions_ta = actions_ta.write(time - 1, action) # Write chosen actions + next_obs, next_prior, next_dangling = get_next_obs_prior_dangling(next_actions_ta, dangling) + next_input = get_input(next_obs) + next_obs_tas = ( # Write OLD observation + obs_tas[0].write(time - 1, obs[0]), # Action inputs + obs_tas[1].write(time - 1, obs[1]), # Parent inputs + obs_tas[2].write(time - 1, obs[2])) # Sibling inputs + next_priors_ta = priors_ta.write(time - 1, prior) # Write OLD prior + finished = next_finished = tf.logical_or( + finished, + time >= max_length) + # When implementing variable length: + # finished = next_finished = tf.logical_or(tf.logical_or( + # finished, # Already finished + # next_dangling == 0), # Currently, this will be 0 not just the first time, but also at max_length + # time >= max_length) + next_lengths = tf.where( + finished, # Ever finished + lengths, + tf.tile(tf.expand_dims(time + 1, 0), [self.batch_size])) + next_loop_state = (next_actions_ta, + next_obs_tas, + next_priors_ta, + next_obs, + next_prior, + next_dangling, + next_lengths, + next_finished) + + return (finished, next_input, next_cell_state, emit_output, next_loop_state) + + # Returns RNN emit outputs TensorArray (i.e. logits), final cell state, and final loop state + with tf.variable_scope('policy'): + _, _, loop_state = tf.nn.raw_rnn(cell=cell, loop_fn=loop_fn) + actions_ta, obs_tas, priors_ta, _, _, _, _, _ = loop_state + + self.actions = tf.transpose(actions_ta.stack(), perm=[1, 0]) # (?, max_length) + self.obs = [tf.transpose(obs_ta.stack(), perm=[1, 0]) for obs_ta in obs_tas] # [(?, max_length)] * 3 + self.priors = tf.transpose(priors_ta.stack(), perm=[1, 0, 2]) # (?, max_length, n_choices) + + + # Generates dictionary containing placeholders needed for a batch of sequences + def make_batch_ph(name): + with tf.name_scope(name): + batch_ph = { + "actions" : tf.placeholder(tf.int32, [None, max_length]), + "obs" : (tf.placeholder(tf.int32, [None, max_length]), + tf.placeholder(tf.int32, [None, max_length]), + tf.placeholder(tf.int32, [None, max_length])), + "priors" : tf.placeholder(tf.float32, [None, max_length, n_choices]), + "lengths" : tf.placeholder(tf.int32, [None,]), + "rewards" : tf.placeholder(tf.float32, [None], name="r") + } + batch_ph = Batch(**batch_ph) + + return batch_ph + + def safe_cross_entropy(p, logq, axis=-1): + safe_logq = tf.where(tf.equal(p, 0.), tf.ones_like(logq), logq) + return - tf.reduce_sum(p * safe_logq, axis) + + # Generates tensor for neglogp of a given batch + def make_neglogp_and_entropy(B): + with tf.variable_scope('policy', reuse=True): + logits, _ = tf.nn.dynamic_rnn(cell=cell, + inputs=get_input(B.obs), + sequence_length=B.lengths, # Backpropagates only through sequence length + dtype=tf.float32) + logits += B.priors + probs = tf.nn.softmax(logits) + logprobs = tf.nn.log_softmax(logits) + + # Generate mask from sequence lengths + # NOTE: Using this mask for neglogp and entropy actually does NOT + # affect training because gradients are zero outside the lengths. + # However, the mask makes tensorflow summaries accurate. + mask = tf.sequence_mask(B.lengths, maxlen=max_length, dtype=tf.float32) + + # Negative log probabilities of sequences + actions_one_hot = tf.one_hot(B.actions, depth=n_choices, axis=-1, dtype=tf.float32) + neglogp_per_step = safe_cross_entropy(actions_one_hot, logprobs, axis=2) # Sum over action dim + neglogp = tf.reduce_sum(neglogp_per_step * mask, axis=1) # Sum over time dim + + # NOTE 1: The above implementation is the same as the one below: + # neglogp_per_step = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,labels=actions) + # neglogp = tf.reduce_sum(neglogp_per_step, axis=1) # Sum over time + # NOTE 2: The above implementation is also the same as the one below, with a few caveats: + # Exactly equivalent when removing priors. + # Equivalent up to precision when including clipped prior. + # Crashes when prior is not clipped due to multiplying zero by -inf. + # neglogp_per_step = -tf.nn.log_softmax(logits + tf.clip_by_value(priors, -2.4e38, 0)) * actions_one_hot + # neglogp_per_step = tf.reduce_sum(neglogp_per_step, axis=2) + # neglogp = tf.reduce_sum(neglogp_per_step, axis=1) # Sum over time + + entropy_per_step = safe_cross_entropy(probs, logprobs, axis=2) # Sum over action dim -> (batch_size, max_length) + entropy = tf.reduce_sum(entropy_per_step * mask, axis=1) # Sum over time dim -> (batch_size, ) + + return neglogp, entropy + + + # On policy batch + self.sampled_batch_ph = make_batch_ph("sampled_batch") + + # Memory batch + self.memory_batch_ph = make_batch_ph("memory_batch") + memory_neglogp, _ = make_neglogp_and_entropy(self.memory_batch_ph) + self.memory_probs = tf.exp(-memory_neglogp) + self.memory_logps = -memory_neglogp + + # PQT batch + if pqt: + self.pqt_batch_ph = make_batch_ph("pqt_batch") + + # Setup losses + with tf.name_scope("losses"): + + neglogp, entropy = make_neglogp_and_entropy(self.sampled_batch_ph) + r = self.sampled_batch_ph.rewards + + # Entropy loss + entropy_loss = -self.entropy_weight * tf.reduce_mean(entropy, name="entropy_loss") + loss = entropy_loss + + # PPO loss + if ppo: + assert not pqt, "PPO is not compatible with PQT" + + self.old_neglogp_ph = tf.placeholder(dtype=tf.float32, shape=(None,), name="old_neglogp") + ratio = tf.exp(self.old_neglogp_ph - neglogp) + clipped_ratio = tf.clip_by_value(ratio, 1. - ppo_clip_ratio, 1. + ppo_clip_ratio) + ppo_loss = -tf.reduce_mean(tf.minimum(ratio * (r - self.baseline), clipped_ratio * (r - self.baseline))) + loss += ppo_loss + + # Define PPO diagnostics + clipped = tf.logical_or(ratio < (1. - ppo_clip_ratio), ratio > 1. + ppo_clip_ratio) + self.clip_fraction = tf.reduce_mean(tf.cast(clipped, tf.float32)) + self.sample_kl = tf.reduce_mean(neglogp - self.old_neglogp_ph) + + # Policy gradient loss + else: + if not pqt or (pqt and pqt_use_pg): + pg_loss = tf.reduce_mean((r - self.baseline) * neglogp, name="pg_loss") + loss += pg_loss + + # Priority queue training loss + if pqt: + pqt_neglogp, _ = make_neglogp_and_entropy(self.pqt_batch_ph) + pqt_loss = pqt_weight * tf.reduce_mean(pqt_neglogp, name="pqt_loss") + loss += pqt_loss + + self.loss = loss + + def make_optimizer(name, learning_rate): + if name == "adam": + return tf.train.AdamOptimizer(learning_rate=learning_rate) + if name == "rmsprop": + return tf.train.RMSPropOptimizer(learning_rate=learning_rate, decay=0.99) + if name == "sgd": + return tf.train.GradientDescentOptimizer(learning_rate=learning_rate) + raise ValueError("Did not recognize optimizer '{}'".format(name)) + + # Create training op + optimizer = make_optimizer(name=optimizer, learning_rate=learning_rate) + with tf.name_scope("train"): + self.grads_and_vars = optimizer.compute_gradients(self.loss) + self.train_op = optimizer.apply_gradients(self.grads_and_vars) + # The two lines above are equivalent to: + # self.train_op = optimizer.minimize(self.loss) + with tf.name_scope("grad_norm"): + self.grads, _ = list(zip(*self.grads_and_vars)) + self.norms = tf.global_norm(self.grads) + + if debug >= 1: + total_parameters = 0 + print("") + for variable in tf.trainable_variables(): + shape = variable.get_shape() + n_parameters = np.product(shape) + total_parameters += n_parameters + print("Variable: ", variable.name) + print(" Shape: ", shape) + print(" Parameters:", n_parameters) + print("Total parameters:", total_parameters) + + # Create summaries + with tf.name_scope("summary"): + if self.summary: + if ppo: + tf.summary.scalar("ppo_loss", ppo_loss) + else: + if not pqt or (pqt and pqt_use_pg): + tf.summary.scalar("pg_loss", pg_loss) + if pqt: + tf.summary.scalar("pqt_loss", pqt_loss) + tf.summary.scalar("entropy_loss", entropy_loss) + tf.summary.scalar("total_loss", self.loss) + tf.summary.scalar("reward", tf.reduce_mean(r)) + tf.summary.scalar("baseline", self.baseline) + tf.summary.histogram("reward", r) + tf.summary.histogram("length", self.sampled_batch_ph.lengths) + for g, v in self.grads_and_vars: + tf.summary.histogram(v.name, v) + tf.summary.scalar(v.name + '_norm', tf.norm(v)) + tf.summary.histogram(v.name + '_grad', g) + tf.summary.scalar(v.name + '_grad_norm', tf.norm(g)) + tf.summary.scalar('gradient norm', self.norms) + self.summaries = tf.summary.merge_all() + + def sample(self, n): + """Sample batch of n expressions""" + + feed_dict = {self.batch_size : n} + + actions, obs, priors = self.sess.run([self.actions, self.obs, self.priors], feed_dict=feed_dict) + + return actions, obs, priors + + + def compute_probs(self, memory_batch, log=False): + """Compute the probabilities of a Batch.""" + + feed_dict = { + self.memory_batch_ph : memory_batch + } + + if log: + fetch = self.memory_logps + else: + fetch = self.memory_probs + probs = self.sess.run([fetch], feed_dict=feed_dict)[0] + return probs + + + def train_step(self, b, sampled_batch, pqt_batch): + """Computes loss, trains model, and returns summaries.""" + + feed_dict = { + self.baseline : b, + self.sampled_batch_ph : sampled_batch + } + + if self.pqt: + feed_dict.update({ + self.pqt_batch_ph : pqt_batch + }) + + if self.ppo: + # Compute old_neglogp to be used for training + old_neglogp = self.sess.run(self.neglogp, feed_dict=feed_dict) + + # Perform multiple epochs of minibatch training + feed_dict[self.old_neglogp_ph] = old_neglogp + indices = np.arange(len(r)) + for epoch in range(self.ppo_n_iters): + self.rng.shuffle(indices) + minibatches = np.array_split(indices, self.ppo_n_mb) + for i, mb in enumerate(minibatches): + mb_feed_dict = {k : v[mb] for k, v in feed_dict.items() if k not in [self.baseline, self.batch_size]} + mb_feed_dict.update({ + self.baseline : b, + self.batch_size : len(mb) + }) + + _ = self.sess.run([self.train_op], feed_dict=mb_feed_dict) + + else: + _ = self.sess.run([self.train_op], feed_dict=feed_dict) + + # Return summaries + if self.summary: + summaries = self.sess.run(self.summaries, feed_dict=feed_dict) + else: + summaries = None + + return summaries diff --git a/dsr/dsr/core.py b/dsr/dsr/core.py new file mode 100644 index 00000000..daec11cd --- /dev/null +++ b/dsr/dsr/core.py @@ -0,0 +1,126 @@ +"""Core deep symbolic optimizer construct.""" + +import json +import zlib +from collections import defaultdict +from multiprocessing import Pool + +import tensorflow as tf + +from dsr.task import set_task +from dsr.controller import Controller +from dsr.train import learn +from dsr.prior import make_prior +from dsr.program import Program + + +class DeepSymbolicOptimizer(): + """ + Deep symbolic optimization model. Includes model hyperparameters and + training configuration. + + Parameters + ---------- + config : dict or str + Config dictionary or path to JSON. See dsr/dsr/config.json for template. + + Attributes + ---------- + config : dict + Configuration parameters for training. + + Methods + ------- + train + Builds and trains the model according to config. + """ + + def __init__(self, config=None): + self.update_config(config) + self.sess = None + + def setup(self, seed=0): + + # Clear the cache, reset the compute graph, and set the seed + Program.clear_cache() + tf.reset_default_graph() + self.seed(seed) # Must be called _after_ resetting graph + + self.pool = self.make_pool() + self.sess = tf.Session() + self.prior = self.make_prior() + self.controller = self.make_controller() + + def train(self, seed=0): + + # Setup the model + self.setup(seed) + + # Train the model + result = learn(self.sess, + self.controller, + self.pool, + **self.config_training) + return result + + def update_config(self, config): + if config is None: + config = {} + elif isinstance(config, str): + with open(config, 'rb') as f: + config = json.load(f) + + self.config = defaultdict(dict, config) + self.config_task = self.config["task"] + self.config_prior = self.config["prior"] + self.config_training = self.config["training"] + self.config_controller = self.config["controller"] + + def seed(self, seed_=0): + """Set the tensorflow seed, which will be offset by a checksum on the + task name to ensure seeds differ across different tasks.""" + + if "name" in self.config_task: + task_name = self.config_task["name"] + else: + task_name = "" + seed_ += zlib.adler32(task_name.encode("utf-8")) + tf.set_random_seed(seed_) + + return seed_ + + def make_prior(self): + prior = make_prior(Program.library, self.config_prior) + return prior + + def make_controller(self): + controller = Controller(self.sess, + self.prior, + **self.config_controller) + return controller + + def make_pool(self): + # Create the pool and set the Task for each worker + pool = None + n_cores_batch = self.config_training.get("n_cores_batch") + if n_cores_batch is not None and n_cores_batch > 1: + pool = Pool(n_cores_batch, + initializer=set_task, + initargs=(self.config_task,)) + + # Set the Task for the parent process + set_task(self.config_task) + + return pool + + def save(self, save_path): + + saver = tf.train.Saver() + saver.save(self.sess, save_path) + + def load(self, load_path): + + if self.sess is None: + self.setup() + saver = tf.train.Saver() + saver.restore(self.sess, load_path) diff --git a/dsr/dsr/cyfunc.pyx b/dsr/dsr/cyfunc.pyx new file mode 100644 index 00000000..11ebd6b5 --- /dev/null +++ b/dsr/dsr/cyfunc.pyx @@ -0,0 +1,90 @@ +''' +# cython: linetrace=True +# distutils: define_macros=CYTHON_TRACE_NOGIL=1 +''' +# Uncomment the above lines for cProfile + +import numpy as np +import array + +# Cython specific C imports +cimport numpy as np +from cpython cimport array +cimport cython +from libc.stdlib cimport malloc, free +from cpython.ref cimport PyObject + +# Static inits +cdef list apply_stack = [[None for i in range(25)] for i in range(1024)] +cdef int *stack_count = malloc(1024 * sizeof(int)) + +@cython.boundscheck(False) # turn off bounds-checking for entire function +@cython.wraparound(False) # turn off negative index wrapping for entire function +def execute(np.ndarray X, int len_traversal, list traversal, int[:] is_input_var): + + """Executes the program according to X. + + Parameters + ---------- + X : array-like, shape = [n_samples, n_features] + Training vectors, where n_samples is the number of samples and + n_features is the number of features. + + Returns + ------- + y_hats : array-like, shape = [n_samples] + The result of executing the program on X. + """ + #sp = 0 # allow a dummy first row, requires a none type function with arity of -1 + + # Init some ints + cdef int sp = -1 # Stack pointer + cdef int Xs = X.shape[0] + + # Give cdef hints for object types + cdef int i + cdef int n + cdef int arity + cdef np.ndarray intermediate_result + cdef list stack_end + cdef object stack_end_function + + for i in range(len_traversal): + + if not is_input_var[i]: + sp += 1 + # Move this to the front with a memset call + stack_count[sp] = 0 + # Store the reference to stack_count[sp] rather than keep calling + apply_stack[sp][stack_count[sp]] = traversal[i] + stack_end = apply_stack[sp] + # The first element is the function itself + stack_end_function = stack_end[0] + arity = stack_end_function.arity + else: + # Not a function, so lazily evaluate later + stack_count[sp] += 1 + stack_end[stack_count[sp]] = X[:, traversal[i].input_var] + + # Keep on doing this so long as arity matches up, we can + # add in numbers above and complete the arity later. + while stack_count[sp] == arity: + intermediate_result = stack_end_function(*stack_end[1:(stack_count[sp] + 1)]) # 85% of overhead + + # I think we can get rid of this line, but will require a major rewrite. + if sp == 0: + return intermediate_result + + sp -= 1 + # Adjust pointer at the end of the stack + stack_end = apply_stack[sp] + stack_count[sp] += 1 + stack_end[stack_count[sp]] = intermediate_result + + # The first element is the function itself + stack_end_function = stack_end[0] + arity = stack_end_function.arity + + # We should never get here + assert False, "Function should never get here!" + return None diff --git a/dsr/dsr/functions.py b/dsr/dsr/functions.py new file mode 100644 index 00000000..705eb602 --- /dev/null +++ b/dsr/dsr/functions.py @@ -0,0 +1,195 @@ +"""Common Tokens used for executable Programs.""" + +import numpy as np +from fractions import Fraction + +from dsr.library import Token, PlaceholderConstant + +GAMMA = 0.57721566490153286060651209008240243104215933593992 + + +"""Define custom unprotected operators""" +def logabs(x1): + """Closure of log for non-positive arguments.""" + return np.log(np.abs(x1)) + +def expneg(x1): + return np.exp(-x1) + +def n3(x1): + return np.power(x1, 3) + +def n4(x1): + return np.power(x1, 4) + +def sigmoid(x1): + return 1 / (1 + np.exp(-x1)) + +def harmonic(x1): + if all(val.is_integer() for val in x1): + return np.array([sum(Fraction(1, d) for d in range(1, int(val)+1)) for val in x1], dtype=np.float32) + else: + return GAMMA + np.log(x1) + 0.5/x1 - 1./(12*x1**2) + 1./(120*x1**4) + + +# Annotate unprotected ops +unprotected_ops = [ + # Binary operators + Token(np.add, "add", arity=2, complexity=1), + Token(np.subtract, "sub", arity=2, complexity=1), + Token(np.multiply, "mul", arity=2, complexity=1), + Token(np.divide, "div", arity=2, complexity=2), + + # Built-in unary operators + Token(np.sin, "sin", arity=1, complexity=3), + Token(np.cos, "cos", arity=1, complexity=3), + Token(np.tan, "tan", arity=1, complexity=4), + Token(np.exp, "exp", arity=1, complexity=4), + Token(np.log, "log", arity=1, complexity=4), + Token(np.sqrt, "sqrt", arity=1, complexity=4), + Token(np.square, "n2", arity=1, complexity=2), + Token(np.negative, "neg", arity=1, complexity=1), + Token(np.abs, "abs", arity=1, complexity=2), + Token(np.maximum, "max", arity=1, complexity=4), + Token(np.minimum, "min", arity=1, complexity=4), + Token(np.tanh, "tanh", arity=1, complexity=4), + Token(np.reciprocal, "inv", arity=1, complexity=2), + + # Custom unary operators + Token(logabs, "logabs", arity=1, complexity=4), + Token(expneg, "expneg", arity=1, complexity=4), + Token(n3, "n3", arity=1, complexity=3), + Token(n4, "n4", arity=1, complexity=3), + Token(sigmoid, "sigmoid", arity=1, complexity=4), + Token(harmonic, "harmonic", arity=1, complexity=4) +] + + +"""Define custom protected operators""" +def protected_div(x1, x2): + with np.errstate(divide='ignore', invalid='ignore', over='ignore'): + return np.where(np.abs(x2) > 0.001, np.divide(x1, x2), 1.) + +def protected_exp(x1): + with np.errstate(over='ignore'): + return np.where(x1 < 100, np.exp(x1), 0.0) + +def protected_log(x1): + """Closure of log for non-positive arguments.""" + with np.errstate(divide='ignore', invalid='ignore'): + return np.where(np.abs(x1) > 0.001, np.log(np.abs(x1)), 0.) + +def protected_sqrt(x1): + """Closure of sqrt for negative arguments.""" + return np.sqrt(np.abs(x1)) + +def protected_inv(x1): + """Closure of inverse for zero arguments.""" + with np.errstate(divide='ignore', invalid='ignore'): + return np.where(np.abs(x1) > 0.001, 1. / x1, 0.) + +def protected_expneg(x1): + with np.errstate(over='ignore'): + return np.where(x1 > -100, np.exp(-x1), 0.0) + +def protected_n2(x1): + with np.errstate(over='ignore'): + return np.where(np.abs(x1) < 1e6, np.square(x1), 0.0) + +def protected_n3(x1): + with np.errstate(over='ignore'): + return np.where(np.abs(x1) < 1e6, np.power(x1, 3), 0.0) + +def protected_n4(x1): + with np.errstate(over='ignore'): + return np.where(np.abs(x1) < 1e6, np.power(x1, 4), 0.0) + +def protected_sigmoid(x1): + return 1 / (1 + protected_expneg(x1)) + +# Annotate protected ops +protected_ops = [ + # Protected binary operators + Token(protected_div, "div", arity=2, complexity=2), + + # Protected unary operators + + Token(protected_exp, "exp", arity=1, complexity=4), + Token(protected_log, "log", arity=1, complexity=4), + Token(protected_log, "logabs", arity=1, complexity=4), # Protected logabs is support, but redundant + Token(protected_sqrt, "sqrt", arity=1, complexity=4), + Token(protected_inv, "inv", arity=1, complexity=2), + Token(protected_expneg, "expneg", arity=1, complexity=4), + Token(protected_n2, "n2", arity=1, complexity=2), + Token(protected_n3, "n3", arity=1, complexity=3), + Token(protected_n4, "n4", arity=1, complexity=3), + Token(protected_sigmoid, "sigmoid", arity=1, complexity=4) +] + +# Add unprotected ops to function map +function_map = { + op.name : op for op in unprotected_ops + } + +# Add protected ops to function map +function_map.update({ + "protected_{}".format(op.name) : op for op in protected_ops + }) + +UNARY_TOKENS = set([op.name for op in function_map.values() if op.arity == 1]) +BINARY_TOKENS = set([op.name for op in function_map.values() if op.arity == 2]) + + +def create_tokens(n_input_var, function_set, protected): + """ + Helper function to create Tokens. + + Parameters + ---------- + n_input_var : int + Number of input variable Tokens. + + function_set : list + Names of registered Tokens, or floats that will create new Tokens. + + protected : bool + Whether to use protected versions of registered Tokens. + """ + + tokens = [] + + # Create input variable Tokens + for i in range(n_input_var): + token = Token(name="x{}".format(i + 1), arity=0, complexity=1, + function=None, input_var=i) + tokens.append(token) + + for op in function_set: + + # Registered Token + if op in function_map: + # Overwrite available protected operators + if protected and not op.startswith("protected_"): + protected_op = "protected_{}".format(op) + if protected_op in function_map: + op = protected_op + + token = function_map[op] + + # Hard-coded floating-point constant + elif isinstance(op, float) or isinstance(op, int): + name = str(op) + value = np.atleast_1d(np.float32(op)) + function = lambda : value + token = Token(name=name, arity=0, complexity=1, function=function) + + # Constant placeholder (to-be-optimized) + elif op == "const": + token = PlaceholderConstant() + + else: + raise ValueError("Operation {} not recognized.".format(op)) + + tokens.append(token) + + return tokens diff --git a/dsr/dsr/library.py b/dsr/dsr/library.py new file mode 100644 index 00000000..e016e4f5 --- /dev/null +++ b/dsr/dsr/library.py @@ -0,0 +1,196 @@ +"""Classes for Token and Library""" + +from collections import defaultdict + +import numpy as np + + +class Token(): + """ + An arbitrary token or "building block" of a Program object. + + Attributes + ---------- + name : str + Name of token. + + arity : int + Arity (number of arguments) of token. + + complexity : float + Complexity of token. + + function : callable + Function associated with the token; used for exectuable Programs. + + input_var : int or None + Index of input if this Token is an input variable, otherwise None. + + Methods + ------- + __call__(input) + Call the Token's function according to input. + """ + + def __init__(self, function, name, arity, complexity, input_var=None): + self.function = function + self.name = name + self.arity = arity + self.complexity = complexity + self.input_var = input_var + + if input_var is not None: + assert function is None, "Input variables should not have functions." + assert arity == 0, "Input variables should have arity zero." + + def __call__(self, *args): + assert self.function is not None, \ + "Token {} is not callable.".format(self.name) + + return self.function(*args) + + def __repr__(self): + return self.name + + +class PlaceholderConstant(Token): + """ + A Token for placeholder constants that will be optimized with respect to + the reward function. The function simply returns the "value" attribute. + + Parameters + ---------- + value : float or None + Current value of the constant, or None if not yet set. + """ + + def __init__(self, value=None): + if value is not None: + value = np.atleast_1d(value) + self.value = value + + def function(): + assert self.value is not None, \ + "Constant is not callable with value None." + return self.value + + super().__init__(function=function, name="const", arity=0, complexity=1) + + def __repr__(self): + if self.value is None: + return self.name + return str(self.value[0]) + + +class Library(): + """ + Library of Tokens. We use a list of Tokens (instead of set or dict) since + we so often index by integers given by the Controller. + + Attributes + ---------- + tokens : list of Token + List of available Tokens in the library. + + names : list of str + Names corresponding to Tokens in the library. + + arities : list of int + Arities corresponding to Tokens in the library. + """ + + def __init__(self, tokens): + + self.tokens = tokens + self.L = len(tokens) + self.names = [t.name for t in tokens] + self.arities = np.array([t.arity for t in tokens], dtype=np.int32) + + self.input_tokens = np.array( + [i for i, t in enumerate(self.tokens) if t.input_var is not None], + dtype=np.int32) + + def get_tokens_of_arity(arity): + _tokens = [i for i in range(self.L) if self.arities[i] == arity] + return np.array(_tokens, dtype=np.int32) + + self.tokens_of_arity = defaultdict(lambda : np.array([], dtype=np.int32)) + for arity in self.arities: + self.tokens_of_arity[arity] = get_tokens_of_arity(arity) + self.terminal_tokens = self.tokens_of_arity[0] + self.unary_tokens = self.tokens_of_arity[1] + self.binary_tokens = self.tokens_of_arity[2] + + try: + self.const_token = self.names.index("const") + except ValueError: + self.const_token = None + self.parent_adjust = np.full_like(self.arities, -1) + count = 0 + for i in range(len(self.arities)): + if self.arities[i] > 0: + self.parent_adjust[i] = count + count += 1 + + trig_names = ["sin", "cos", "tan", "csc", "sec", "cot"] + trig_names += ["arc" + name for name in trig_names] + + self.float_tokens = np.array( + [i for i, t in enumerate(self.tokens) if t.arity == 0 and t.input_var is None], + dtype=np.int32) + self.trig_tokens = np.array( + [i for i, t in enumerate(self.tokens) if t.name in trig_names], + dtype=np.int32) + + inverse_tokens = { + "inv" : "inv", + "neg" : "neg", + "exp" : "log", + "log" : "exp", + "sqrt" : "n2", + "n2" : "sqrt" + } + token_from_name = {t.name : i for i, t in enumerate(self.tokens)} + self.inverse_tokens = {token_from_name[k] : token_from_name[v] for k, v in inverse_tokens.items() if k in token_from_name and v in token_from_name} + + def __getitem__(self, val): + """Shortcut to get Token by name or index.""" + + if isinstance(val, str): + try: + i = self.names.index(val) + except ValueError: + raise TokenNotFoundError("Token {} does not exist.".format(val)) + elif isinstance(val, (int, np.integer)): + i = val + else: + raise TokenNotFoundError("Library must be indexed by str or int, not {}.".format(type(val))) + + try: + token = self.tokens[i] + except IndexError: + raise TokenNotFoundError("Token index {} does not exist".format(i)) + return token + + def tokenize(self, inputs): + """Convert inputs to list of Tokens.""" + + if isinstance(inputs, str): + inputs = inputs.split(',') + elif not isinstance(inputs, list) and not isinstance(inputs, np.ndarray): + inputs = [inputs] + tokens = [input_ if isinstance(input_, Token) else self[input_] for input_ in inputs] + return tokens + + def actionize(self, inputs): + """Convert inputs to array of 'actions', i.e. ints corresponding to + Tokens in the Library.""" + + tokens = self.tokenize(inputs) + actions = np.array([self.tokens.index(t) for t in tokens], + dtype=np.int32) + return actions + + +class TokenNotFoundError(Exception): + pass diff --git a/dsr/dsr/memory.py b/dsr/dsr/memory.py new file mode 100644 index 00000000..88c8eb0d --- /dev/null +++ b/dsr/dsr/memory.py @@ -0,0 +1,358 @@ +"""Classes for memory buffers, priority queues, and quantile estimation.""" + +import heapq +from collections import namedtuple + +import numpy as np + + +Batch = namedtuple( + "Batch", ["actions", "obs", "priors", "lengths", "rewards"]) + + +def make_queue(controller=None, priority=False, capacity=np.inf, seed=0): + """Factory function for various Queues. + + Parameters + ---------- + controller : dsr.controller.Controller + Reference to the Controller, used to compute probabilities of items in + the Queue. + + priority : bool + If True, returns an object inheriting UniquePriorityQueue. Otherwise, + returns an object inheriting from UniqueQueue. + + capacity : int + Maximum queue length. + + seed : int + RNG seed used for random sampling. + + Returns + ------- + queue : ProgramQueue + Dynamic class inheriting from ProgramQueueMixin and a Queue subclass. + """ + + if priority: + Base = UniquePriorityQueue + else: + Base = UniqueQueue + + class ProgramQueue(ProgramQueueMixin, Base): + def __init__(self, controller, capacity, seed): + ProgramQueueMixin.__init__(self, controller) + Base.__init__(self, capacity, seed) + + queue = ProgramQueue(controller, capacity, seed) + return queue + + +def get_samples(batch, key): + """ + Returns a sub-Batch with samples from the given indices. + + Parameters + ---------- + key : int or slice + Indices of samples to return. + + Returns + ------- + batch : Batch + Sub-Batch with samples from the given indices. + """ + + batch = Batch( + actions=batch.actions[key], + obs=tuple(o[key] for o in batch.obs), + priors=batch.priors[key], + lengths=batch.lengths[key], + rewards=batch.rewards[key]) + return batch + + +# Adapted from https://github.com/tensorflow/models/blob/1af55e018eebce03fb61bba9959a04672536107d/research/brain_coder/common/utils.py +class ItemContainer(object): + """Class for holding an item with its score. + + Defines a comparison function for use in the heap-queue. + """ + + def __init__(self, score, item, extra_data): + self.item = item + self.score = score + self.extra_data = extra_data + + def __lt__(self, other): + assert isinstance(other, type(self)) + return self.score < other.score + + def __eq__(self, other): + assert isinstance(other, type(self)) + return self.item == other.item + + def __iter__(self): + """Allows unpacking like a tuple.""" + yield self.score + yield self.item + yield self.extra_data + + def __repr__(self): + """String representation of this item. + + `extra_data` is not included in the representation. We are assuming that + `extra_data` is not easily interpreted by a human (if it was, it should be + hashable, like a string or tuple). + + Returns: + String representation of `self`. + """ + return str((self.score, self.item)) + + def __str__(self): + return repr(self) + + +class Queue(object): + """Abstract class for queue that must define a push and pop routine""" + + def __init__(self, capacity, seed=0): + self.capacity = capacity + self.rng = np.random.RandomState(seed) + self.heap = [] + self.unique_items = set() + + def push(self, score, item, extra_data): + raise NotImplementedError + + def pop(self): + raise NotImplementedError + + def random_sample(self, sample_size): + """Uniform randomly select items from the queue. + + Args: + sample_size: Number of random samples to draw. The same item can be + sampled multiple times. + + Returns: + List of sampled items (of length `sample_size`). Each element in the list + is a tuple: (item, extra_data). + """ + idx = self.rng.choice(len(self.heap), sample_size, ) + return [(self.heap[i].item, self.heap[i].extra_data) for i in idx] + + def __len__(self): + return len(self.heap) + + def __iter__(self): + for _, item, _ in self.heap: + yield item + + def __repr__(self): + return '[' + ', '.join(repr(c) for c in self.heap) + ']' + + def __str__(self): + return repr(self) + + +class UniqueQueue(Queue): + """A queue in which duplicates are not allowed. Instead, adding a duplicate + moves that item to the back of the queue.""" + + def push(self, score, item, extra_data=None): + """Push an item onto the queue, or move it to the back if already + present. + + Score is unused but included as an argument to follow the interface. + """ + + container = ItemContainer(None, item, extra_data) + + # If the item is already in the queue, move it to the back of the queue + # and return + if item in self.unique_items: + self.heap.remove(container) + self.heap.append(container) + return + + # If the queue is at capacity, first pop the front of the queue + if len(self.heap) >= self.capacity: + self.pop() + + # Add the item + self.heap.append(container) + self.unique_items.add(item) + + def pop(self): + """Pop the front of the queue (the oldest item).""" + + if not self.heap: + return () + score, item, extra_data = self.heap.pop(0) + self.unique_items.remove(item) + return (score, item, extra_data) + + +# Adapted from https://github.com/tensorflow/models/blob/1af55e018eebce03fb61bba9959a04672536107d/research/brain_coder/common/utils.py +class UniquePriorityQueue(Queue): + """A priority queue where duplicates are not added. + + The top items by score remain in the queue. When the capacity is reached, + the lowest scored item in the queue will be dropped. + """ + + def push(self, score, item, extra_data=None): + """Push an item onto the queue. + + If the queue is at capacity, the item with the smallest score will be + dropped. Note that it is assumed each item has exactly one score. The same + item with a different score will still be dropped. + + Args: + score: Number used to prioritize items in the queue. Largest scores are + kept in the queue. + item: A hashable item to be stored. Duplicates of this item will not be + added to the queue. + extra_data: An extra (possible not hashable) data to store with the item. + """ + if item in self.unique_items: + return + if len(self.heap) >= self.capacity: + _, popped_item, _ = heapq.heappushpop( + self.heap, ItemContainer(score, item, extra_data)) + self.unique_items.add(item) + self.unique_items.remove(popped_item) + else: + heapq.heappush(self.heap, ItemContainer(score, item, extra_data)) + self.unique_items.add(item) + + def pop(self): + """Pop the item with the lowest score. + + Returns: + score: Item's score. + item: The item that was popped. + extra_data: Any extra data stored with the item. + """ + if not self.heap: + return () + score, item, extra_data = heapq.heappop(self.heap) + self.unique_items.remove(item) + return score, item, extra_data + + def get_max(self): + """Peek at the item with the highest score. + + Returns: + Same as `pop`. + """ + if not self.heap: + return () + score, item, extra_data = heapq.nlargest(1, self.heap)[0] + return score, item, extra_data + + def get_min(self): + """Peek at the item with the lowest score. + + Returns: + Same as `pop`. + """ + if not self.heap: + return () + score, item, extra_data = heapq.nsmallest(1, self.heap)[0] + return score, item, extra_data + + def iter_in_order(self): + """Iterate over items in the queue from largest score to smallest. + + Yields: + item: Hashable item. + extra_data: Extra data stored with the item. + """ + for _, item, extra_data in heapq.nlargest(len(self.heap), self.heap): + yield item, extra_data + + +class ProgramQueueMixin(): + """A mixin for Queues with additional utilities specific to Batch and + Program.""" + + def __init__(self, controller=None): + self.controller = controller + + def push_sample(self, sample, program): + """ + Push a single sample corresponding to Program to the queue. + + Parameters + ---------- + sample : Batch + A Batch comprising a single sample. + + program : Program + Program corresponding to the sample. + """ + + id_ = program.str + score = sample.rewards + self.push(score, id_, sample) + + def push_batch(self, batch, programs): + """Push a Batch corresponding to Programs to the queue.""" + + for i, program in enumerate(programs): + sample = get_samples(batch, i) + self.push_sample(sample, program) + + def push_best(self, batch, programs): + """Push the single best sample from a Batch""" + + i = np.argmax(batch.rewards) + sample = get_samples(batch, i) + program = programs[i] + self.push_sample(sample, program) + + def sample_batch(self, sample_size): + """Randomly select items from the queue and return them as a Batch.""" + + assert len(self.heap) > 0, "Cannot sample from an empty queue." + samples = [sample for (id_, sample) in self.random_sample(sample_size)] + batch = self._make_batch(samples) + return batch + + def _make_batch(self, samples): + """Turns a list of samples into a Batch.""" + + actions = np.stack([s.actions for s in samples], axis=0) + obs = tuple([np.stack([s.obs[i] for s in samples], axis=0) for i in range(3)]) + priors = np.stack([s.priors for s in samples], axis=0) + lengths = np.array([s.lengths for s in samples], dtype=np.int32) + rewards = np.array([s.rewards for s in samples], dtype=np.float32) + batch = Batch(actions=actions, obs=obs, priors=priors, + lengths=lengths, rewards=rewards) + return batch + + def to_batch(self): + """Return the entire queue as a Batch.""" + + samples = [container.extra_data for container in self.heap] + batch = self._make_batch(samples) + return batch + + def compute_probs(self): + """Computes the probabilities of items in the queue according to the + Controller.""" + + if self.controller is None: + raise RuntimeError("Cannot compute probabilities. This Queue does \ + not have a Controller.") + return self.controller.compute_probs(self.to_batch()) + + def get_rewards(self): + """Returns the rewards""" + + r = [container.extra_data.rewards for container in self.heap] + return r diff --git a/dsr/dsr/prior.py b/dsr/dsr/prior.py new file mode 100644 index 00000000..510d79f8 --- /dev/null +++ b/dsr/dsr/prior.py @@ -0,0 +1,527 @@ +"""Class for Prior object.""" + +import numpy as np + +from dsr.subroutines import ancestors +from dsr.library import TokenNotFoundError + + +def make_prior(library, config_prior): + """Factory function for JointPrior object.""" + + prior_dict = { + "relational" : RelationalConstraint, + "length" : LengthConstraint, + "repeat" : RepeatConstraint, + "inverse" : InverseUnaryConstraint, + "trig" : TrigConstraint, + "const" : ConstConstraint + } + + priors = [] + warnings = [] + for prior_type, prior_args in config_prior.items(): + assert prior_type in prior_dict, \ + "Unrecognized prior type: {}.".format(prior_type) + prior_class = prior_dict[prior_type] + + if isinstance(prior_args, dict): + prior_args = [prior_args] + for single_prior_args in prior_args: + + # Attempt to build the Prior. Any Prior can fail if it references a + # Token not in the Library. + try: + prior = prior_class(library, **single_prior_args) + warning = prior.validate() + except TokenNotFoundError: + prior = None + warning = "Uses Tokens not in the Library." + + # Add warning context + if warning is not None: + warning = "Skipping invalid '{}' with arguments {}. " \ + "Reason: {}" \ + .format(prior_class.__name__, single_prior_args, warning) + warnings.append(warning) + + # Add the Prior if there are no warnings + if warning is None: + priors.append(prior) + + joint_prior = JointPrior(library, priors) + + print("-- Building prior -------------------") + print("\n".join(["WARNING: " + message for message in warnings])) + print(joint_prior.describe()) + print("-------------------------------------") + + return joint_prior + + +class JointPrior(): + """A collection of joint Priors.""" + + def __init__(self, library, priors): + """ + Parameters + ---------- + library : Library + The Library associated with the Priors. + + priors : list of Prior + The individual Priors to be joined. + """ + + self.library = library + self.L = self.library.L + self.priors = priors + assert all([prior.library is library for prior in priors]), \ + "All Libraries must be identical." + + self.requires_parents_siblings = True + + self.describe() + + def initial_prior(self): + combined_prior = np.zeros((self.L,), dtype=np.float32) + for prior in self.priors: + combined_prior += prior.initial_prior() + return combined_prior + + def __call__(self, actions, parent, sibling, dangling): + zero_prior = np.zeros((actions.shape[0], self.L), dtype=np.float32) + ind_priors = [zero_prior.copy() for _ in range(len(self.priors))] + for i in range(len(self.priors)): + ind_priors[i] += self.priors[i](actions, parent, sibling, dangling) + combined_prior = sum(ind_priors) + zero_prior + return combined_prior + + def describe(self): + message = "\n".join(prior.describe() for prior in self.priors) + return message + + +class Prior(): + """Abstract class whose call method return logits.""" + + def __init__(self, library): + self.library = library + self.L = library.L + + def validate(self): + """ + Determine whether the Prior has a valid configuration. This is useful + when other algorithmic parameters may render the Prior degenerate. For + example, having a TrigConstraint with no trig Tokens. + + Returns + ------- + message : str or None + Error message if Prior is invalid, or None if it is valid. + """ + + return None + + def init_zeros(self, actions): + """Helper function to generate a starting prior of zeros.""" + + batch_size = actions.shape[0] + prior = np.zeros((batch_size, self.L), dtype=np.float32) + return prior + + def initial_prior(self): + """ + Compute the initial prior, before any actions are selected. + + Returns + ------- + initial_prior : array + Initial logit adjustment before actions are selected. Shape is + (self.L,) as it will be broadcast to batch size later. + """ + + return np.zeros((self.L,), dtype=np.float32) + + def __call__(self, actions, parent, sibling, dangling): + """ + Compute the prior (logit adjustment) given the current actions. + + Returns + ------- + prior : array + Logit adjustment for selecting next action. Shape is (batch_size, + self.L). + """ + + raise NotImplementedError + + def describe(self): + """Describe the Prior.""" + + message = "No description." + return message + + +class Constraint(Prior): + def __init__(self, library): + Prior.__init__(self, library) + + def make_constraint(self, mask, tokens): + """ + Generate the prior for a batch of constraints and the corresponding + Tokens to constrain. + + For example, with L=5 and tokens=[1,2], a constrained row of the prior + will be: [0.0, -np.inf, -np.inf, 0.0, 0.0]. + + Parameters + __________ + + mask : np.ndarray, shape=(?,), dtype=np.bool_ + Boolean mask of samples to constrain. + + tokens : np.ndarray, dtype=np.int32 + Tokens to constrain. + + Returns + _______ + + prior : np.ndarray, shape=(?, L), dtype=np.float32 + Logit adjustment. Since these are hard constraints, each element is + either 0.0 or -np.inf. + """ + + prior = np.zeros((mask.shape[0], self.L), dtype=np.float32) + for t in tokens: + prior[mask, t] = -np.inf + return prior + + +class RelationalConstraint(Constraint): + """ + Class that constrains the following: + + Constrain (any of) `targets` from being the `relationship` of (any of) + `effectors`. + + Parameters + ---------- + targets : list of Tokens + List of Tokens, all of which will be constrained if any of effectors + are the given relationship. + + effectors : list of Tokens + List of Tokens, any of which will cause all targets to be constrained + if they are the given relationship. + + relationship : choice of ["child", "descendant", "sibling", "uchild"] + The type of relationship to constrain. + """ + + def __init__(self, library, targets, effectors, relationship): + Prior.__init__(self, library) + self.targets = library.actionize(targets) + self.effectors = library.actionize(effectors) + self.relationship = relationship + + def validate(self): + message = [] + if self.relationship in ["child", "descendant", "uchild"]: + if np.isin(self.effectors, self.library.terminal_tokens).any(): + message = "{} relationship cannot have terminal effectors." \ + .format(self.relationship.capitalize()) + return message + if len(self.targets) == 0: + message = "There are no target Tokens." + return message + if len(self.effectors) == 0: + message = "There are no effector Tokens." + return message + return None + + def __call__(self, actions, parent, sibling, dangling): + + if self.relationship == "descendant": + mask = ancestors(actions=actions, + arities=self.library.arities, + ancestor_tokens=self.effectors) + prior = self.make_constraint(mask, self.targets) + + elif self.relationship == "child": + parents = self.effectors + adj_parents = self.library.parent_adjust[parents] + mask = np.isin(parent, adj_parents) + prior = self.make_constraint(mask, self.targets) + + elif self.relationship == "sibling": + # The sibling relationship is reflexive: if A is a sibling of B, + # then B is also a sibling of A. Thus, we combine two priors, where + # targets and effectors are swapped. + mask = np.isin(sibling, self.effectors) + prior = self.make_constraint(mask, self.targets) + mask = np.isin(sibling, self.targets) + prior += self.make_constraint(mask, self.effectors) + + elif self.relationship == "uchild": + # Case 1: parent is a unary effector + unary_effectors = np.intersect1d(self.effectors, + self.library.unary_tokens) + adj_unary_effectors = self.library.parent_adjust[unary_effectors] + mask = np.isin(parent, adj_unary_effectors) + # Case 2: sibling is a target and parent is an effector + adj_effectors = self.library.parent_adjust[self.effectors] + mask += np.logical_and(np.isin(sibling, self.targets), + np.isin(parent, adj_effectors)) + prior = self.make_constraint(mask, [self.targets]) + + return prior + + def describe(self): + + targets = ", ".join([self.library.names[t] for t in self.targets]) + effectors = ", ".join([self.library.names[t] for t in self.effectors]) + relationship = { + "child" : "a child", + "sibling" : "a sibling", + "descendant" : "a descendant", + "uchild" : "the only unique child" + }[self.relationship] + message = "[{}] cannot be {} of [{}]." \ + .format(targets, relationship, effectors) + return message + + +class TrigConstraint(RelationalConstraint): + """Class that constrains trig Tokens from being the desendants of trig + Tokens.""" + + def __init__(self, library): + targets = library.trig_tokens + effectors = library.trig_tokens + RelationalConstraint.__init__(self, library, + targets=targets, + effectors=effectors, + relationship="descendant") + + +class ConstConstraint(RelationalConstraint): + """Class that constrains the const Token from being the only unique child + of all non-terminal Tokens.""" + + def __init__(self, library): + targets = library.const_token + effectors = np.concatenate([library.unary_tokens, + library.binary_tokens]) + RelationalConstraint.__init__(self, library, + targets=targets, + effectors=effectors, + relationship="uchild") + + +class InverseUnaryConstraint(Constraint): + """Class that constrains each unary Token from being the child of its + corresponding inverse unary Tokens.""" + + def __init__(self, library): + Prior.__init__(self, library) + self.priors = [] + for target, effector in library.inverse_tokens.items(): + targets = [target] + effectors = [effector] + prior = RelationalConstraint(library, + targets=targets, + effectors=effectors, + relationship="child") + self.priors.append(prior) + + def validate(self): + if len(self.priors) == 0: + message = "There are no inverse unary Token pairs in the Library." + return message + return None + + def __call__(self, actions, parent, sibling, dangling): + prior = sum([prior(actions, parent, sibling, dangling) + for prior in self.priors]) + return prior + + def describe(self): + message = [prior.describe() for prior in self.priors] + return "\n".join(message) + + +class RepeatConstraint(Constraint): + """Class that constrains Tokens to appear between a minimum and/or maximum + number of times.""" + + def __init__(self, library, tokens, min_=None, max_=None): + """ + Parameters + ---------- + tokens : Token or list of Tokens + Token(s) which should, in total, occur between min_ and max_ times. + + min_ : int or None + Minimum number of times tokens should occur. + + max_ : int or None + Maximum number of times tokens should occur. + """ + + Prior.__init__(self, library) + assert min_ is not None or max_ is not None, \ + "At least one of (min_, max_) must not be None." + self.min = min_ + self.max = max_ + self.tokens = library.actionize(tokens) + + assert min_ is None, "Repeat minimum constraints are not yet " \ + "supported. This requires knowledge of length constraints." + + def __call__(self, actions, parent, sibling, dangling): + counts = np.sum(np.isin(actions, self.tokens), axis=1) + prior = self.init_zeros(actions) + if self.min is not None: + raise NotImplementedError + if self.max is not None: + mask = counts >= self.max + prior += self.make_constraint(mask, self.tokens) + return prior + + def describe(self): + names = ", ".join([self.library.names[t] for t in self.tokens]) + if self.min is None: + message = "[{}] cannot occur more than {} times."\ + .format(names, self.max) + elif self.max is None: + message = "[{}] must occur at least {} times."\ + .format(names, self.min) + else: + message = "[{}] must occur between {} and {} times."\ + .format(names, self.min, self.max) + return message + + +class LengthConstraint(Constraint): + """Class that constrains the Program from falling within a minimum and/or + maximum length""" + + def __init__(self, library, min_=None, max_=None): + """ + Parameters + ---------- + min_ : int or None + Minimum length of the Program. + + max_ : int or None + Maximum length of the Program. + """ + + Prior.__init__(self, library) + self.min = min_ + self.max = max_ + + assert min_ is not None or max_ is not None, \ + "At least one of (min_, max_) must not be None." + + def initial_prior(self): + prior = Prior.initial_prior(self) + for t in self.library.terminal_tokens: + prior[t] = -np.inf + return prior + + def __call__(self, actions, parent, sibling, dangling): + + # Initialize the prior + prior = self.init_zeros(actions) + i = actions.shape[1] - 1 # Current time + + # Never need to constrain max length for first half of expression + if self.max is not None and (i + 2) >= self.max // 2: + remaining = self.max - (i + 1) + # assert sum(dangling > remaining) == 0, (dangling, remaining) + mask = dangling >= remaining - 1 # Constrain binary + prior += self.make_constraint(mask, self.library.binary_tokens) + mask = dangling == remaining # Constrain unary + prior += self.make_constraint(mask, self.library.unary_tokens) + + # Constrain terminals when dangling == 1 until selecting the + # (min_length)th token + if self.min is not None and (i + 2) < self.min: + mask = dangling == 1 # Constrain terminals + prior += self.make_constraint(mask, self.library.terminal_tokens) + + return prior + + def describe(self): + message = [] + if self.min is not None: + message.append("Sequences have minimum length {}.".format(self.min)) + if self.max is not None: + message.append("Sequences have maximum length {}.".format(self.max)) + message = "\n".join(message) + return message + + +class UniformArityPrior(Prior): + """Class that puts a fixed prior on arities by transforming the initial + distribution from uniform over tokens to uniform over arities.""" + + def __init__(self, library): + + Prior.__init__(self, library) + + # For each token, subtract log(n), where n is the total number of tokens + # in the library with the same arity as that token. This is equivalent + # to... For each arity, subtract log(n) from tokens of that arity, where + # n is the total number of tokens of that arity + self.logit_adjust = np.zeros((self.L,), dtype=np.float32) + for arity, tokens in self.library.tokens_of_arity.items(): + self.logit_adjust[tokens] -= np.log(len(tokens)) + + def initial_prior(self): + return self.logit_adjust + + def __call__(self, actions, parent, sibling, dangling): + + # This will be broadcast when added to the joint prior + prior = self.logit_adjust + return prior + + +class SoftLengthPrior(Prior): + """Class the puts a soft prior on length. Before loc, terminal probabilities + are scaled by exp(-(t - loc) ** 2 / (2 * scale)) where dangling == 1. After + loc, non-terminal probabilities are scaled by that number.""" + + def __init__(self, library, loc, scale): + + Prior.__init__(self, library) + + self.loc = loc + self.scale = scale + + self.terminal_mask = np.zeros((self.L,), dtype=np.bool) + self.terminal_mask[self.library.terminal_tokens] = True + + self.nonterminal_mask = ~self.terminal_mask + + def __call__(self, actions, parent, sibling, dangling): + + # Initialize the prior + prior = self.init_zeros(actions) + t = actions.shape[1] # Current time + + # Adjustment to terminal or non-terminal logits + logit_adjust = -(t - self.loc) ** 2 / (2 * self.scale) + + # Before loc, decrease p(terminal) where dangling == 1 + if t < self.loc: + prior[dangling == 1] += self.terminal_mask * logit_adjust + + # After loc, decrease p(non-terminal) + else: + prior += self.nonterminal_mask * logit_adjust + + return prior diff --git a/dsr/dsr/program.py b/dsr/dsr/program.py new file mode 100644 index 00000000..d00f3e30 --- /dev/null +++ b/dsr/dsr/program.py @@ -0,0 +1,640 @@ +"""Class for symbolic expression object or program.""" + +import array +import os +import warnings +from textwrap import indent + +import numpy as np +from sympy.parsing.sympy_parser import parse_expr +from sympy import pretty + +from dsr.functions import PlaceholderConstant +from dsr.const import make_const_optimizer +from dsr.utils import cached_property +import dsr.utils as U + + +def _finish_tokens(tokens): + """ + Complete the pre-order traversal. + + Parameters + ---------- + tokens : list of integers + A list of integers corresponding to tokens in the library. The list + defines an expression's pre-order traversal. + + Returns + _______ + tokens : list of integers + A list of integers corresponding to tokens in the library. The list + defines an expression's pre-order traversal. "Dangling" programs are + completed with repeated "x1" until the expression completes. + + """ + + arities = np.array([Program.library.arities[t] for t in tokens]) + dangling = 1 + np.cumsum(arities - 1) + + if 0 in dangling: + expr_length = 1 + np.argmax(dangling == 0) + tokens = tokens[:expr_length] + else: + # Extend with first variable until complete + tokens = np.append(tokens, np.random.choice(Program.library.input_tokens, size=dangling[-1])) + + return tokens + + +def from_str_tokens(str_tokens, optimize, skip_cache=False): + """ + Memoized function to generate a Program from a list of str and/or float. + See from_tokens() for details. + + Parameters + ---------- + str_tokens : str | list of (str | float) + Either a comma-separated string of tokens and/or floats, or a list of + str and/or floats. + + optimize : bool + See from_tokens(). + + skip_cache : bool + See from_tokens(). + + Returns + ------- + program : Program + See from_tokens(). + """ + + # Convert str to list of str + if isinstance(str_tokens, str): + str_tokens = str_tokens.split(",") + + # Convert list of str|float to list of tokens + if isinstance(str_tokens, list): + traversal = [] + constants = [] + for s in str_tokens: + if s in Program.library.names: + t = Program.library.names.index(s.lower()) + elif U.is_float(s): + assert "const" not in str_tokens, "Currently does not support both placeholder and hard-coded constants." + assert not optimize, "Currently does not support optimization with hard-coded constants." + t = Program.library.const_token + constants.append(float(s)) + else: + raise ValueError("Did not recognize token {}.".format(s)) + traversal.append(t) + traversal = np.array(traversal, dtype=np.int32) + else: + raise ValueError("Input must be list or string.") + + # Generate base Program (with "const" for constants) + p = from_tokens(traversal, optimize=optimize, skip_cache=skip_cache) + + # Replace any constants + p.set_constants(constants) + + return p + + +def from_tokens(tokens, optimize, skip_cache=False): + """ + Memoized function to generate a Program from a list of tokens. + + Since some tokens are nonfunctional, this first computes the corresponding + traversal. If that traversal exists in the cache, the corresponding Program + is returned. Otherwise, a new Program is returned. + + Parameters + ---------- + tokens : list of integers + A list of integers corresponding to tokens in the library. The list + defines an expression's pre-order traversal. "Dangling" programs are + completed with repeated "x1" until the expression completes. + + optimize : bool + Whether to optimize the program before returning it. + + skip_cache : bool + Whether to bypass the cache when creating the program. + + Returns + _______ + program : Program + The Program corresponding to the tokens, either pulled from memoization + or generated from scratch. + """ + + ''' + Truncate expressions that complete early; extend ones that don't complete + ''' + tokens = _finish_tokens(tokens) + + # For stochastic Tasks, there is no cache; always generate a new Program. + # For deterministic Programs, if the Program is in the cache, return it; + # otherwise, create a new one and add it to the cache. + if skip_cache: + p = Program(tokens, optimize=optimize) + elif Program.task.stochastic: + p = Program(tokens, optimize=optimize) + else: + key = tokens.tostring() + if key in Program.cache: + p = Program.cache[key] + p.count += 1 + else: + p = Program(tokens, optimize=optimize) + Program.cache[key] = p + + return p + + +class Program(object): + """ + The executable program representing the symbolic expression. + + The program comprises unary/binary operators, constant placeholders + (to-be-optimized), input variables, and hard-coded constants. + + Parameters + ---------- + tokens : list of integers + A list of integers corresponding to tokens in the library. "Dangling" + programs are completed with repeated "x1" until the expression + completes. + + optimize : bool + Whether to optimize the program upon initializing it. + + Attributes + ---------- + traversal : list + List of operators (type: Function) and terminals (type: int, float, or + str ("const")) encoding the pre-order traversal of the expression tree. + + tokens : np.ndarry (dtype: int) + Array of integers whose values correspond to indices + + const_pos : list of int + A list of indicies of constant placeholders along the traversal. + + float_pos : list of float + A list of indices of constants placeholders or floating-point constants + along the traversal. + + sympy_expr : str + The (lazily calculated) SymPy expression corresponding to the program. + Used for pretty printing _only_. + + base_r : float + The base reward (reward without penalty) of the program on the training + data. + + complexity : float + The (lazily calcualted) complexity of the program. + + r : float + The (lazily calculated) reward of the program on the training data. + + count : int + The number of times this Program has been sampled. + + str : str + String representation of tokens. Useful as unique identifier. + """ + + # Static variables + task = None # Task + library = None # Library + const_optimizer = None # Function to optimize constants + cache = {} + + # Cython-related static variables + have_cython = None # Do we have cython installed + execute = None # Link to execute. Either cython or python + cyfunc = None # Link to cyfunc lib since we do an include inline + + def __init__(self, tokens, optimize): + + """ + Builds the Program from a list of Tokens, optimizes the Constants + against reward function, and evalutes the reward. + """ + + self.traversal = [Program.library[t] for t in tokens] + self.const_pos = [i for i, t in enumerate(tokens) if Program.library[t].name == "const"] # Just constant placeholder positions + self.len_traversal = len(self.traversal) + + if self.have_cython and self.len_traversal > 1: + self.is_input_var = array.array('i', [t.input_var is not None for t in self.traversal]) + + self.invalid = False + self.str = tokens.tostring() + + if optimize: + _ = self.optimize() + + self.count = 1 + + def cython_execute(self, X): + """Executes the program according to X using Cython. + + Parameters + ---------- + X : array-like, shape = [n_samples, n_features] + Training vectors, where n_samples is the number of samples and + n_features is the number of features. + + Returns + ------- + y_hats : array-like, shape = [n_samples] + The result of executing the program on X. + """ + + if self.len_traversal > 1: + return self.cyfunc.execute(X, self.len_traversal, self.traversal, self.is_input_var) + else: + return self.python_execute(X) + + def python_execute(self, X): + """Executes the program according to X using Python. + + Parameters + ---------- + X : array-like, shape = [n_samples, n_features] + Training vectors, where n_samples is the number of samples and + n_features is the number of features. + + Returns + ------- + y_hats : array-like, shape = [n_samples] + The result of executing the program on X. + """ + + # # Check for single-node programs + # node = self.traversal[0] + # if isinstance(node, float): + # return np.repeat(node, X.shape[0]) + # if isinstance(node, int): + # return X[:, node] + + apply_stack = [] + + for node in self.traversal: + + apply_stack.append([node]) + + while len(apply_stack[-1]) == apply_stack[-1][0].arity + 1: + # Apply functions that have sufficient arguments + token = apply_stack[-1][0] + terminals = apply_stack[-1][1:] + # terminals = [np.repeat(t, X.shape[0]) if isinstance(t, float) + # else X[:, t] if isinstance(t, int) + # else t for t in apply_stack[-1][1:]] + if token.input_var is not None: + intermediate_result = X[:, token.input_var] + else: + intermediate_result = token(*terminals) + if len(apply_stack) != 1: + apply_stack.pop() + apply_stack[-1].append(intermediate_result) + else: + return intermediate_result + + # We should never get here + assert False, "Function should never get here!" + return None + + + def optimize(self): + """ + Optimizes the constant tokens against the training data and returns the + optimized constants. + + This function generates an objective function based on the training + dataset, reward function, and constant optimizer. It ignores penalties + because the Program structure is fixed, thus penalties are all the same. + It then optimizes the constants of the program and returns the optimized + constants. + + Returns + _______ + optimized_constants : vector + Array of optimized constants. + """ + + # Create the objective function, which is a function of the constants being optimized + def f(consts): + self.set_constants(consts) + r = self.task.reward_function(self) + obj = -r # Constant optimizer minimizes the objective function + + # Need to reset to False so that a single invalid call during + # constant optimization doesn't render the whole Program invalid. + self.invalid = False + + return obj + + assert self.execute is not None, "set_execute needs to be called first" + + if len(self.const_pos) > 0: + # Do the optimization + x0 = np.ones(len(self.const_pos)) # Initial guess + optimized_constants = Program.const_optimizer(f, x0) + self.set_constants(optimized_constants) + + else: + # No need to optimize if there are no constants + optimized_constants = [] + + return optimized_constants + + def set_constants(self, consts): + """Sets the program's constants to the given values""" + + for i, const in enumerate(consts): + # Create a new instance of PlaceholderConstant instead of changing + # the "values" attribute, otherwise all Programs will have the same + # instance and just overwrite each other's value. + self.traversal[self.const_pos[i]] = PlaceholderConstant(const) + + @classmethod + def clear_cache(cls): + """Clears the class' cache""" + + cls.cache = {} + + @classmethod + def set_task(cls, task): + """Sets the class' Task""" + + Program.task = task + Program.library = task.library + + @classmethod + def set_const_optimizer(cls, name, **kwargs): + """Sets the class' constant optimizer""" + + const_optimizer = make_const_optimizer(name, **kwargs) + Program.const_optimizer = const_optimizer + + @classmethod + def set_complexity_penalty(cls, name, weight): + """Sets the class' complexity penalty""" + + all_functions = { + # No penalty + None : lambda p : 0.0, + + # Length of tree + "length" : lambda p : len(p) + } + + assert name in all_functions, "Unrecognzied complexity penalty name" + + if weight == 0: + Program.complexity_penalty = lambda p : 0.0 + else: + Program.complexity_penalty = lambda p : weight * all_functions[name](p) + + @classmethod + def set_execute(cls, protected): + """Sets which execute method to use""" + + """ + If cython ran, we will have a 'c' file generated. The dynamic libary can be + given different names, so it's not reliable for testing if cython ran. + """ + cpath = os.path.join(os.path.dirname(__file__),'cyfunc.c') + + if os.path.isfile(cpath): + from . import cyfunc + Program.cyfunc = cyfunc + execute_function = Program.cython_execute + Program.have_cython = True + else: + execute_function = Program.python_execute + Program.have_cython = False + + if protected: + Program.execute = execute_function + else: + + class InvalidLog(): + """Log class to catch and record numpy warning messages""" + + def __init__(self): + self.error_type = None # One of ['divide', 'overflow', 'underflow', 'invalid'] + self.error_node = None # E.g. 'exp', 'log', 'true_divide' + self.new_entry = False # Flag for whether a warning has been encountered during a call to Program.execute() + + def write(self, message): + """This is called by numpy when encountering a warning""" + + if not self.new_entry: # Only record the first warning encounter + message = message.strip().split(' ') + self.error_type = message[1] + self.error_node = message[-1] + self.new_entry = True + + def update(self, p): + """If a floating-point error was encountered, set Program.invalid + to True and record the error type and error node.""" + + if self.new_entry: + p.invalid = True + p.error_type = self.error_type + p.error_node = self.error_node + self.new_entry = False + + + invalid_log = InvalidLog() + np.seterrcall(invalid_log) # Tells numpy to call InvalidLog.write() when encountering a warning + + # Define closure for execute function + def unsafe_execute(p, X): + """This is a wrapper for execute_function. If a floating-point error + would be hit, a warning is logged instead, p.invalid is set to True, + and the appropriate nan/inf value is returned. It's up to the task's + reward function to decide how to handle nans/infs.""" + + with np.errstate(all='log'): + y = execute_function(p, X) + invalid_log.update(p) + return y + + Program.execute = unsafe_execute + + + @cached_property + def complexity(self): + """Evaluates and returns the complexity of the program""" + + return Program.complexity_penalty(self.traversal) + + + @cached_property + def base_r(self): + """Evaluates and returns the base reward of the program on the training + set""" + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + + return self.task.reward_function(self) + + @cached_property + def r(self): + """Evaluates and returns the reward of the program on the training + set""" + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + + return self.base_r - self.complexity + + + @cached_property + def evaluate(self): + """Evaluates and returns the evaluation metrics of the program.""" + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + + return self.task.evaluate(self) + + @cached_property + def complexity_eureqa(self): + """Computes sum of token complexity based on Eureqa complexity measures.""" + + complexity = sum([t.complexity for t in self.traversal]) + return complexity + + + @cached_property + def sympy_expr(self): + """ + Returns the attribute self.sympy_expr. + + This is actually a bit complicated because we have to go: traversal --> + tree --> serialized tree --> SymPy expression + """ + + tree = self.traversal.copy() + tree = build_tree(tree) + tree = convert_to_sympy(tree) + try: + expr = parse_expr(tree.__repr__()) # SymPy expression + except: + expr = "N/A" + + return expr + + + def pretty(self): + """Returns pretty printed string of the program""" + return pretty(self.sympy_expr) + + + def print_stats(self): + """Prints the statistics of the program""" + print("\tReward: {}".format(self.r)) + print("\tBase reward: {}".format(self.base_r)) + print("\tCount: {}".format(self.count)) + print("\tInvalid: {}".format(self.invalid)) + print("\tTraversal: {}".format(self)) + print("\tExpression:") + print("{}\n".format(indent(self.pretty(), '\t '))) + + + def __repr__(self): + """Prints the program's traversal""" + + return ','.join([repr(t) for t in self.traversal]) + + +############################################################################### +# Everything below this line is currently only being used for pretty printing # +############################################################################### + + +# Possible library elements that sympy capitalizes +capital = ["add", "mul", "pow"] + + +class Node(object): + """Basic tree class supporting printing""" + + def __init__(self, val): + self.val = val + self.children = [] + + def __repr__(self): + children_repr = ",".join(repr(child) for child in self.children) + if len(self.children) == 0: + return self.val # Avoids unnecessary parantheses, e.g. x1() + return "{}({})".format(self.val, children_repr) + + +def build_tree(traversal): + """Recursively builds tree from pre-order traversal""" + + op = traversal.pop(0) + n_children = op.arity + val = repr(op) + if val in capital: + val = val.capitalize() + + node = Node(val) + + for _ in range(n_children): + node.children.append(build_tree(traversal)) + + return node + + +def convert_to_sympy(node): + """Adjusts trees to only use node values supported by sympy""" + + if node.val == "div": + node.val = "Mul" + new_right = Node("Pow") + new_right.children.append(node.children[1]) + new_right.children.append(Node("-1")) + node.children[1] = new_right + + elif node.val == "sub": + node.val = "Add" + new_right = Node("Mul") + new_right.children.append(node.children[1]) + new_right.children.append(Node("-1")) + node.children[1] = new_right + + elif node.val == "inv": + node.val = Node("Pow") + node.children.append(Node("-1")) + + elif node.val == "neg": + node.val = Node("Mul") + node.children.append(Node("-1")) + + elif node.val == "n2": + node.val = "Pow" + node.children.append(Node("2")) + + elif node.val == "n3": + node.val = "Pow" + node.children.append(Node("3")) + + elif node.val == "n4": + node.val = "Pow" + node.children.append(Node("4")) + + for child in node.children: + convert_to_sympy(child) + + + + return node diff --git a/dsr/dsr/run.py b/dsr/dsr/run.py new file mode 100644 index 00000000..94e2b65b --- /dev/null +++ b/dsr/dsr/run.py @@ -0,0 +1,224 @@ +"""Parallelized, single-point launch script to run DSR or GP on a set of benchmarks.""" + +import warnings +warnings.filterwarnings('ignore', category=DeprecationWarning) +warnings.filterwarnings('ignore', category=FutureWarning) + +import os +import sys +import json +import time +from datetime import datetime +import multiprocessing +from functools import partial +from pkg_resources import resource_filename +import zlib + +import click +import numpy as np +import pandas as pd +from sympy.parsing.sympy_parser import parse_expr +from sympy import srepr + +from dsr import DeepSymbolicOptimizer +from dsr.program import Program +from dsr.task.regression.dataset import BenchmarkDataset +from dsr.baselines import gpsr + + +def train_dsr(name_and_seed, config): + """Trains DSR and returns dict of reward, expression, and traversal""" + + # Override the benchmark name and output file + name, seed = name_and_seed + config["task"]["name"] = name + config["training"]["output_file"] = "dsr_{}_{}.csv".format(name, seed) + + # Try importing TensorFlow (with suppressed warnings), Controller, and learn + # When parallelizing across tasks, these will already be imported, hence try/except + try: + os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' + import tensorflow as tf + tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR) + from dsr.controller import Controller + from dsr.train import learn + except ModuleNotFoundError: # Specific subclass of ImportError for when module is not found, probably needs to be excepted first + print("One or more libraries not found") + raise ModuleNotFoundError + except ImportError: + # Have we already imported tf? If so, this is the error we want to dodge. + if 'tf' in globals(): + pass + else: + raise ImportError + + # Train the model + model = DeepSymbolicOptimizer(config) + start = time.time() + result = {"name" : name, "seed" : seed} # Name and seed are listed first + result.update(model.train(seed=seed)) + result["t"] = time.time() - start + result.pop("program") + + return result + + +def train_gp(name_and_seed, logdir, config_task, config_gp): + """Trains GP and returns dict of reward, expression, and program""" + + name, seed = name_and_seed + config_gp["seed"] = seed + zlib.adler32(name.encode("utf-8")) + + start = time.time() + + # Load the dataset + config_dataset = config_task["dataset"] + config_dataset["name"] = name + dataset = BenchmarkDataset(**config_dataset) + + # Fit the GP + gp = gpsr.GP(dataset=dataset, **config_gp) + p, logbook = gp.train() + + # Retrieve results + r = base_r = p.fitness.values[0] + str_p = str(p) + nmse_test = gp.nmse_test(p)[0] + nmse_test_noiseless = gp.nmse_test_noiseless(p)[0] + success = gp.success(p) + + # Many failure cases right now for converting to SymPy expression + try: + expression = repr(parse_expr(str_p.replace("X", "x").replace("add", "Add").replace("mul", "Mul"))) + except: + expression = "N/A" + + # Save run details + drop = ["gen", "nevals"] + df_fitness = pd.DataFrame(logbook.chapters["fitness"]).drop(drop, axis=1) + df_fitness = df_fitness.rename({"avg" : "fit_avg", "min" : "fit_min"}, axis=1) + df_fitness["fit_best"] = df_fitness["fit_min"].cummin() + df_len = pd.DataFrame(logbook.chapters["size"]).drop(drop, axis=1) + df_len = df_len.rename({"avg" : "l_avg"}, axis=1) + df = pd.concat([df_fitness, df_len], axis=1, sort=False) + df.to_csv(os.path.join(logdir, "gp_{}_{}.csv".format(name, seed)), index=False) + + result = { + "name" : name, + "seed" : seed, + "r" : r, + "base_r" : base_r, + "nmse_test" : nmse_test, + "nmse_test_noiseless" : nmse_test_noiseless, + "success" : success, + "expression" : expression, + "traversal" : str_p, + "t" : time.time() - start + } + + return result + + +@click.command() +@click.argument('config_template', default="config.json") +@click.option('--method', default="dsr", type=click.Choice(["dsr", "gp"]), help="Symbolic regression method") +@click.option('--mc', default=1, type=int, help="Number of Monte Carlo trials for each benchmark") +@click.option('--output_filename', default=None, help="Filename to write results") +@click.option('--n_cores_task', '--n', default=1, help="Number of cores to spread out across tasks") +@click.option('--seed_shift', default=0, type=int, help="Integer to add to each seed (i.e. to combine multiple runs)") +@click.option('--b', multiple=True, type=str, help="Name of benchmark or benchmark prefix") +def main(config_template, method, mc, output_filename, n_cores_task, seed_shift, b): + """Runs DSR or GP on multiple benchmarks using multiprocessing.""" + + # Load the config file + with open(config_template, encoding='utf-8') as f: + config = json.load(f) + + # Required configs + config_task = config["task"] # Task specification parameters + config_training = config["training"] # Training hyperparameters + + # Optional configs + config_controller = config.get("controller") # Controller hyperparameters + config_language_model_prior = config.get("language_model_prior") # Language model hyperparameters + config_gp = config.get("gp") # GP hyperparameters + + # Create output directories + if output_filename is None: + output_filename = "benchmark_{}.csv".format(method) + config_training["logdir"] = os.path.join( + config_training["logdir"], + "log_{}".format(datetime.now().strftime("%Y-%m-%d-%H%M%S"))) + logdir = config_training["logdir"] + if "dataset" in config_task and "backup" in config_task["dataset"] and config_task["dataset"]["backup"]: + config_task["dataset"]["logdir"] = logdir + os.makedirs(logdir, exist_ok=True) + output_filename = os.path.join(logdir, output_filename) + # Use benchmark name from config if not specified as command-line arg + if len(b) == 0: + if isinstance(config_task["name"], str): + b = (config_task["name"],) + elif isinstance(config_task["name"], list): + b = tuple(config_task["name"]) + + # Shortcut to run all Nguyen benchmarks + benchmarks = list(b) + if "Nguyen" in benchmarks: + benchmarks.remove("Nguyen") + benchmarks += ["Nguyen-{}".format(i+1) for i in range(12)] + + # Generate benchmark-seed pairs for each MC. When passed to the TF RNG, + # seeds will be added to checksums on the benchmark names + unique_benchmarks = benchmarks.copy() + benchmarks *= mc + seeds = (np.arange(mc) + seed_shift).repeat(len(unique_benchmarks)).tolist() + names_and_seeds = list(zip(benchmarks, seeds)) + + # Edit n_cores_task and/or n_cores_batch + if n_cores_task == -1: + n_cores_task = multiprocessing.cpu_count() + if n_cores_task > len(benchmarks): + print("Setting 'n_cores_task' to {} for batch because there are only {} benchmarks.".format(len(benchmarks), len(benchmarks))) + n_cores_task = len(benchmarks) + if method == "dsr": + if config_training["verbose"] and n_cores_task > 1: + print("Setting 'verbose' to False for parallelized run.") + config_training["verbose"] = False + if config_training["n_cores_batch"] != 1 and n_cores_task > 1: + print("Setting 'n_cores_batch' to 1 to avoid nested child processes.") + config_training["n_cores_batch"] = 1 + print("Running {} for n={} on benchmarks {}".format(method, mc, unique_benchmarks)) + + # Write terminal command and config.json into log directory + cmd_filename = os.path.join(logdir, "cmd.out") + with open(cmd_filename, 'w') as f: + print(" ".join(sys.argv), file=f) + config_filename = os.path.join(logdir, "config.json") + with open(config_filename, 'w') as f: + json.dump(config, f, indent=4) + + # Define the work + if method == "dsr": + work = partial(train_dsr, config=config) + elif method == "gp": + work = partial(train_gp, logdir=logdir, config_task=config_task, config_gp=config_gp) + + # Farm out the work + write_header = True + if n_cores_task > 1: + pool = multiprocessing.Pool(n_cores_task) + for result in pool.imap_unordered(work, names_and_seeds): + pd.DataFrame(result, index=[0]).to_csv(output_filename, header=write_header, mode='a', index=False) + print("Completed {} ({} of {}) in {:.0f} s".format(result["name"], result["seed"]+1-seed_shift, mc, result["t"])) + write_header = False + else: + for name_and_seed in names_and_seeds: + result = work(name_and_seed) + pd.DataFrame(result, index=[0]).to_csv(output_filename, header=write_header, mode='a', index=False) + write_header = False + + print("Results saved to: {}".format(output_filename)) + + +if __name__ == "__main__": + main() diff --git a/dsr/dsr/subroutines.py b/dsr/dsr/subroutines.py new file mode 100644 index 00000000..fbe4221a --- /dev/null +++ b/dsr/dsr/subroutines.py @@ -0,0 +1,120 @@ +"""Numba-compiled subroutines used for deep symbolic optimization.""" + +from numba import jit, prange +import numpy as np + + +@jit(nopython=True, parallel=True) +def parents_siblings(tokens, arities, parent_adjust): + """ + Given a batch of action sequences, computes and returns the parents and + siblings of the next element of the sequence. + + The batch has shape (N, L), where N is the number of sequences (i.e. batch + size) and L is the length of each sequence. In some cases, expressions may + already be complete; in these cases, this function sees the start of a new + expression, even though the return value for these elements won't matter + because their gradients will be zero because of sequence_length. + + Parameters + __________ + + tokens : np.ndarray, shape=(N, L), dtype=np.int32 + Batch of action sequences. Values correspond to library indices. + + arities : np.ndarray, dtype=np.int32 + Array of arities corresponding to library indices. + + parent_adjust : np.ndarray, dtype=np.int32 + Array of parent sub-library index corresponding to library indices. + + Returns + _______ + + adj_parents : np.ndarray, shape=(N,), dtype=np.int32 + Adjusted parents of the next element of each action sequence. + + siblings : np.ndarray, shape=(N,), dtype=np.int32 + Siblings of the next element of each action sequence. + + """ + N, L = tokens.shape + + empty_parent = np.max(parent_adjust) + 1 # Empty token is after all non-empty tokens + empty_sibling = len(arities) # Empty token is after all non-empty tokens + adj_parents = np.full(shape=(N,), fill_value=empty_parent, dtype=np.int32) + siblings = np.full(shape=(N,), fill_value=empty_sibling, dtype=np.int32) + # Parallelized loop over action sequences + for r in prange(N): + arity = arities[tokens[r, -1]] + if arity > 0: # Parent is the previous element; no sibling + adj_parents[r] = parent_adjust[tokens[r, -1]] + continue + dangling = 0 + # Loop over elements in an action sequence + for c in range(L): + arity = arities[tokens[r, L - c - 1]] + dangling += arity - 1 + if dangling == 0: # Parent is L-c-1, sibling is the next + adj_parents[r] = parent_adjust[tokens[r, L - c - 1]] + siblings[r] = tokens[r, L - c] + break + return adj_parents, siblings + + +@jit(nopython=True, parallel=True) +def ancestors(actions, arities, ancestor_tokens): + """ + Given a batch of action sequences, determines whether the next element of + the sequence has an ancestor in ancestor_tokens. + + The batch has shape (N, L), where N is the number of sequences (i.e. batch + size) and L is the length of each sequence. In some cases, expressions may + already be complete; in these cases, this function sees the start of a new + expression, even though the return value for these elements won't matter + because their gradients will be zero because of sequence_length. + + Parameters + __________ + + actions : np.ndarray, shape=(N, L), dtype=np.int32 + Batch of action sequences. Values correspond to library indices. + + arities : np.ndarray, dtype=np.int32 + Array of arities corresponding to library indices. + + ancestor_tokens : np.ndarray, dtype=np.int32 + Array of ancestor library indices to check. + + Returns + _______ + + mask : np.ndarray, shape=(N,), dtype=np.bool_ + Mask of whether the next element of each sequence has an ancestor in + ancestor_tokens. + """ + + N, L = actions.shape + mask = np.zeros(shape=(N,), dtype=np.bool_) + # Parallelized loop over action sequences + for r in prange(N): + dangling = 0 + threshold = None # If None, current branch does not have trig ancestor + for c in range(L): + arity = arities[actions[r, c]] + dangling += arity - 1 + # Turn "on" if a trig function is found + # Remain "on" until branch completes + if threshold is None: + for trig_token in ancestor_tokens: + if actions[r, c] == trig_token: + threshold = dangling - 1 + break + # Turn "off" once the branch completes + else: + if dangling == threshold: + threshold = None + # If the sequences ended "on", then there is a trig ancestor + if threshold is not None: + mask[r] = True + return mask diff --git a/dsr/dsr/task/__init__.py b/dsr/dsr/task/__init__.py new file mode 100644 index 00000000..8dc70998 --- /dev/null +++ b/dsr/dsr/task/__init__.py @@ -0,0 +1 @@ +from dsr.task.task import make_task, set_task, Task diff --git a/dsr/dsr/task/regression/__init__.py b/dsr/dsr/task/regression/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dsr/dsr/task/regression/benchmarks.csv b/dsr/dsr/task/regression/benchmarks.csv new file mode 100644 index 00000000..3a2ce53f --- /dev/null +++ b/dsr/dsr/task/regression/benchmarks.csv @@ -0,0 +1,38 @@ +name,variables,expression,train_spec,test_spec,function_set +Nguyen-1,1,"pow(x1,3)+pow(x1,2)+x1","{""all"":{""U"":[-1,1,20]}}",None,Koza +Nguyen-2,1,"pow(x1,4)+pow(x1,3)+pow(x1,2)+x1","{""all"":{""U"":[-1,1,20]}}",None,Koza +Nguyen-3,1,"pow(x1,5)+pow(x1,4)+pow(x1,3)+pow(x1,2)+x1","{""all"":{""U"":[-1,1,20]}}",None,Koza +Nguyen-4,1,"pow(x1,6)+pow(x1,5)+pow(x1,4)+pow(x1,3)+pow(x1,2)+x1","{""all"":{""U"":[-1,1,20]}}",None,Koza +Nguyen-5,1,"sin(pow(x1,2))*cos(x1)-1","{""all"":{""U"":[-1,1,20]}}",None,Koza +Nguyen-6,1,"sin(x1)+sin(x1+pow(x1,2))","{""all"":{""U"":[-1,1,20]}}",None,Koza +Nguyen-7,1,"log(x1+1)+log(pow(x1,2)+1)","{""all"":{""U"":[0,2,20]}}",None,Koza +Nguyen-8,1,sqrt(x1),"{""all"":{""U"":[0,4,20]}}",None,Koza +Nguyen-9,2,"sin(x1)+sin(pow(x2,2))","{""all"":{""U"":[0,1,20]}}",None,Koza +Nguyen-10,2,2*sin(x1)*cos(x2),"{""all"":{""U"":[0,1,20]}}",None,Koza +Nguyen-11,2,"pow(x1,x2)","{""all"":{""U"":[0,1,20]}}",None,Koza +Nguyen-12,2,"pow(x1,4)-pow(x1,3)+div(pow(x2,2),2)-x2","{""all"":{""U"":[0,1,20]}}",None,Koza +Nguyen-2a,1,"4*pow(x1,4)+3*pow(x1,3)+2*pow(x1,2)+x1","{""all"":{""U"":[-1,1,20]}}",None,Koza +Nguyen-5a,1,"sin(pow(x1,2))*cos(x1)-2","{""all"":{""U"":[-1,1,20]}}",None,Koza +Nguyen-8a,1,"pow(x1,1/3)","{""all"":{""U"":[0,4,20]}}",None,Koza +Nguyen-8aa,1,"pow(x1,2/3)","{""all"":{""U"":[0,4,20]}}",None,Koza +Nguyen-1c,1,"3.39*pow(x1,3)+2.12*pow(x1,2)+1.78*x1","{""all"":{""U"":[-1,1,20]}}",None,CKoza +Nguyen-5c,1,"sin(pow(x1,2))*cos(x1)-0.75","{""all"":{""U"":[-1,1,20]}}",None,CKoza +Nguyen-7c,1,"log(x1+1.4)+log(pow(x1,2)+1.3)","{""all"":{""U"":[0,2,20]}}",None,CKoza +Nguyen-8c,1,sqrt(1.23*x1),"{""all"":{""U"":[0,4,20]}}",None,CKoza +Nguyen-10c,2,sin(1.5*x1)*cos(0.5*x2),"{""all"":{""U"":[0,1,20]}}",None,CKoza +GrammarVAE-1,1,"1./3+x1+sin(pow(x1,2))","{""all"":{""E"":[-10,10,1000]}}",None,GrammarVAE +Jin-1,2,"2.5*pow(x1,4)-1.3*pow(x1,3)+0.5*pow(x2,2)-1.7*x2","{""all"":{""U"":[-3.0,3.0,100]}}","{""all"":{""U"":[-3.0,3.0,30]}}",Jin +Jin-2,2,"8.0*pow(x1,2)+8.0*pow(x2,3)-15.0","{""all"":{""U"":[-3.0,3.0,100]}}","{""all"":{""U"":[-3.0,3.0,30]}}",Jin +Jin-3,2,"0.2*pow(x1,3)+0.5*pow(x2,3)-1.2*x2-0.5*x1","{""all"":{""U"":[-3.0,3.0,100]}}","{""all"":{""U"":[-3.0,3.0,30]}}",Jin +Jin-4,2,1.5*exp(x1)+5.0*cos(x2),"{""all"":{""U"":[-3.0,3.0,100]}}","{""all"":{""U"":[-3.0,3.0,30]}}",Jin +Jin-5,2,6.0*sin(x1)*cos(x2),"{""all"":{""U"":[-3.0,3.0,100]}}","{""all"":{""U"":[-3.0,3.0,30]}}",Jin +Jin-6,2,1.35*x1*x2+5.5*sin((x1-1.0)*(x2-1.0)),"{""all"":{""U"":[-3.0,3.0,100]}}","{""all"":{""U"":[-3.0,3.0,30]}}",Jin +Neat-1,1,"pow(x1,4)+pow(x1,3)+pow(x1,2)+x1","{""all"":{""U"":[-1,1,20]}}",None,KozaPlus1 +Neat-2,1,"pow(x1,5)+pow(x1,4)+pow(x1,3)+pow(x1,2)+x1","{""all"":{""U"":[-1,1,20]}}",None,KozaPlus1 +Neat-3,1,"sin(pow(x1,2))*cos(x1)-1","{""all"":{""U"":[-1,1,20]}}",None,KozaPlus1 +Neat-4,1,"log(x1+1)+log(pow(x1,2)+1)","{""all"":{""U"":[0,2,20]}}",None,KozaPlus1 +Neat-5,2,2*sin(x1)*cos(x2),"{""all"":{""U"":[-1,1,100]}}",None,Koza +Neat-6,1,harmonic(x1),"{""all"":{""E"":[1,50,1]}}","{""all"":{""E"":[1,120,1]}}",KeijzerPlus1 +Neat-7,2,2-2.1*cos(9.8*x1)*sin(1.3*x2),"{""all"":{""U"":[-50,50,10000]}}",None,Korns +Neat-8,2,"div(exp(-pow(x1-1,2)),(1.2+pow((x2-2.5),2)))","{""all"":{""U"":[0.3,4,100]}}",None,Vladislavleva-B +Neat-9,2,"div(1,(1+pow(x1,-4)))+div(1,(1+pow(x2,-4)))","{""all"":{""E"":[-5,5,0.4]}}",None,Koza diff --git a/dsr/dsr/task/regression/dataset.py b/dsr/dsr/task/regression/dataset.py new file mode 100644 index 00000000..a6c9dfe6 --- /dev/null +++ b/dsr/dsr/task/regression/dataset.py @@ -0,0 +1,274 @@ +"""Class for deterministically generating a benchmark dataset from benchmark specifications.""" + +import os +import ast +import itertools +from pkg_resources import resource_filename +import zlib + +import click +import pandas as pd +import numpy as np + +from dsr.functions import function_map + + +class BenchmarkDataset(object): + """ + Class used to generate (X, y) data from a named benchmark expression. + + Parameters + ---------- + name : str + Name of benchmark expression. + + benchmark_source : str, optional + Filename of CSV describing benchmark expressions. + + root : str, optional + Directory containing benchmark_source and function_sets.csv. + + noise : float, optional + If not None, Gaussian noise is added to the y values with standard + deviation = noise * RMS of the noiseless y training values. + + dataset_size_multiplier : float, optional + Multiplier for size of the dataset. + + seed : int, optional + Random number seed used to generate data. Checksum on name is added to + seed. + + logdir : str, optional + Directory where experiment logfiles are saved. + + backup : bool, optional + Save generated dataset in logdir if logdir is provided. + """ + + def __init__(self, name, benchmark_source="benchmarks.csv", root=None, noise=0.0, + dataset_size_multiplier=1.0, seed=0, logdir=None, + backup=False): + # Set class variables + self.name = name + self.seed = seed + self.noise = noise if noise is not None else 0.0 + self.dataset_size_multiplier = dataset_size_multiplier if dataset_size_multiplier is not None else 1.0 + + # Set random number generator used for sampling X values + seed += zlib.adler32(name.encode("utf-8")) # Different seed for each name, otherwise two benchmarks with the same domain will always have the same X values + self.rng = np.random.RandomState(seed) + + # Load benchmark data + if root is None: + root = resource_filename("dsr.task", "regression") + benchmark_path = os.path.join(root, benchmark_source) + benchmark_df = pd.read_csv(benchmark_path, index_col=0, encoding="ISO-8859-1") + row = benchmark_df.loc[name] + self.n_input_var = row["variables"] + + # Create symbolic expression + self.numpy_expr = self.make_numpy_expr(row["expression"]) + + # Create X values + train_spec = ast.literal_eval(row["train_spec"]) + test_spec = ast.literal_eval(row["test_spec"]) + if test_spec is None: + test_spec = train_spec + self.X_train = self.make_X(train_spec) + self.X_test = self.make_X(test_spec) + self.train_spec = train_spec + self.test_spec = test_spec + + # Compute y values + self.y_train = self.numpy_expr(self.X_train) + self.y_test = self.numpy_expr(self.X_test) + self.y_train_noiseless = self.y_train.copy() + self.y_test_noiseless = self.y_test.copy() + + # Add Gaussian noise + if self.noise > 0: + y_rms = np.sqrt(np.mean(self.y_train**2)) + scale = self.noise * y_rms + self.y_train += self.rng.normal(loc=0, scale=scale, size=self.y_train.shape) + self.y_test += self.rng.normal(loc=0, scale=scale, size=self.y_test.shape) + elif self.noise < 0: + print('WARNING: Ignoring negative noise value: {}'.format(self.noise)) + + # Load default function set + function_set_path = os.path.join(root, "function_sets.csv") + function_set_df = pd.read_csv(function_set_path, index_col=0) + function_set_name = row["function_set"] + self.function_set = function_set_df.loc[function_set_name].tolist()[0].strip().split(',') + + # Prepare status output + output_message = '\n-- Building dataset -----------------\n' + output_message += 'Benchmark path : {}\n'.format(benchmark_path) + output_message += 'Generated data for benchmark : {}\n'.format(name) + output_message += 'Function set path : {}\n'.format(function_set_path) + output_message += 'Function set : {} --> {}\n'.format(function_set_name, self.function_set) + if backup and logdir is not None: + output_message += self.save(logdir) + output_message += '-------------------------------------\n\n' + print(output_message) + + def make_X(self, spec): + """Creates X values based on specification""" + + features = [] + for i in range(1, self.n_input_var + 1): + + # Hierarchy: "all" --> "x{}".format(i) + input_var = "x{}".format(i) + if "all" in spec: + input_var = "all" + elif input_var not in spec: + input_var = "x1" + + if "U" in spec[input_var]: + low, high, n = spec[input_var]["U"] + n = int(n * self.dataset_size_multiplier) + feature = self.rng.uniform(low=low, high=high, size=n) + elif "E" in spec[input_var]: + start, stop, step = spec[input_var]["E"] + if step > stop - start: + n = step + else: + n = int((stop - start)/step) + 1 + n = int(n * self.dataset_size_multiplier) + feature = np.linspace(start=start, stop=stop, num=n, endpoint=True) + else: + raise ValueError("Did not recognize specification for {}: {}.".format(input_var, spec[input_var])) + features.append(feature) + + # Do multivariable combinations + if "E" in spec[input_var] and self.n_input_var > 1: + X = np.array(list(itertools.product(*features))) + else: + X = np.column_stack(features) + + return X + + def make_numpy_expr(self, s): + # This isn't pretty, but unlike sympy's lambdify, this ensures we use + # our protected functions. Otherwise, some expressions may have large + # error even if the functional form is correct due to the training set + # not using protected functions. + + # Replace function names + s = s.replace("ln(", "log(") + s = s.replace("pi", "np.pi") + s = s.replace("pow", "np.power") + for k in function_map.keys(): + s = s.replace(k + '(', "function_map['{}'].function(".format(k)) + + # Replace variable names + for i in reversed(range(self.n_input_var)): + old = "x{}".format(i+1) + new = "x[:, {}]".format(i) + s = s.replace(old, new) + + numpy_expr = lambda x : eval(s) + + return numpy_expr + + def save(self, logdir='./'): + save_path = os.path.join(logdir,'data_{}_n{:.2f}_d{:.0f}_s{}.csv'.format( + self.name, self.noise, self.dataset_size_multiplier, self.seed)) + try: + os.makedirs(logdir, exist_ok=True) + np.savetxt( + save_path, + np.concatenate( + ( + np.hstack((self.X_train, self.y_train[..., np.newaxis])), + np.hstack((self.X_test, self.y_test[..., np.newaxis])) + ), axis=0), + delimiter=',', fmt='%1.5f' + ) + return 'Saved dataset to : {}\n'.format(save_path) + except: + import sys + e = sys.exc_info()[0] + print("WARNING: Could not save dataset: {}".format(e)) + + def plot(self, logdir='./'): + """Plot Dataset with underlying ground truth.""" + if self.X_train.shape[1] == 1: + from matplotlib import pyplot as plt + save_path = os.path.join(logdir,'plot_{}_n{:.2f}_d{:.0f}_s{}.png'.format( + self.name, self.noise, self.dataset_size_multiplier, self.seed)) + + # Draw ground truth expression + bounds = list(list(self.train_spec.values())[0].values())[0][:2] + x = np.linspace(bounds[0], bounds[1], endpoint=True, num=100) + y = self.numpy_expr(x[:, None]) + plt.plot(x, y, color='red', linestyle='dashed') + # Draw the actual points + plt.scatter(self.X_train, self.y_train) + # Add a title + plt.title( + "{} N:{} M:{} S:{}".format( + self.name, self.noise, self.dataset_size_multiplier, self.seed), + fontsize=7) + try: + os.makedirs(logdir, exist_ok=True) + plt.savefig(save_path) + print('Saved plot to : {}'.format(save_path)) + except: + import sys + e = sys.exc_info()[0] + print("WARNING: Could not plot dataset: {}".format(e)) + plt.close() + else: + print("WARNING: Plotting only supported for 2D datasets.") + + +@click.command() +@click.argument("benchmark_source", default="benchmarks.csv") +@click.option('--plot', is_flag=True) +@click.option('--save_csv', is_flag=True) +@click.option('--sweep', is_flag=True) +def main(benchmark_source, plot, save_csv, sweep): + """Plots all benchmark expressions.""" + + regression_path = resource_filename("dsr.task", "regression/") + benchmark_path = os.path.join(regression_path, benchmark_source) + save_dir = os.path.join(regression_path, 'log') + df = pd.read_csv(benchmark_path, encoding="ISO-8859-1") + names = df["name"].to_list() + for name in names: + + if not name.startswith("Nguyen") and not name.startswith("Constant") and not name.startswith("Custom"): + continue + + datasets = [] + + # Noiseless + d = BenchmarkDataset( + name=name, + benchmark_source=benchmark_source) + datasets.append(d) + + # Generate all combinations of noise levels and dataset size multipliers + if sweep and name.startswith("Nguyen"): + noises = [0.0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.10] + dataset_size_multipliers = [1.0, 10.0] + for noise in noises: + for dataset_size_multiplier in dataset_size_multipliers: + d = BenchmarkDataset( + name=name, + benchmark_source=benchmark_source, + noise=noise, + dataset_size_multiplier=dataset_size_multiplier, + backup=save_csv, + logdir=save_dir) + datasets.append(d) + + # Plot and/or save datasets + for dataset in datasets: + if plot and dataset.X_train.shape[1] == 1: + dataset.plot(save_dir) + +if __name__ == "__main__": + main() diff --git a/dsr/dsr/task/regression/function_sets.csv b/dsr/dsr/task/regression/function_sets.csv new file mode 100644 index 00000000..8c23cdd3 --- /dev/null +++ b/dsr/dsr/task/regression/function_sets.csv @@ -0,0 +1,13 @@ +name,function_set +Koza,"add,sub,mul,div,sin,cos,exp,log" +CKoza,"add,sub,mul,div,sin,cos,exp,log,const" +KozaPlus1,"add,sub,mul,div,sin,cos,exp,log,1.0" +Korns,"add,sub,mul,div,sin,cos,exp,log,n2,n3,sqrt,tan,tanh,const" +Keijzer,"add,mul,inv,neg,sqrt,const" +KeijzerPlus1,"add,mul,inv,neg,sqrt,1.0,const" +Vladislavleva-A,"add,sub,mul,div,n2" +Vladislavleva-B,"add,sub,mul,div,n2,exp,expneg" +Vladislavleva-C,"add,sub,mul,div,n2,exp,expneg,sin,cos" +None,"add,sub,mul,div,sin,cos,exp,log" +Jin,"add,sub,mul,div,sin,cos,exp,n2,n3,const" +GrammarVAE,"add,mul,div,sin,exp,1.0,2.0,3.0" diff --git a/dsr/dsr/task/regression/regression.py b/dsr/dsr/task/regression/regression.py new file mode 100644 index 00000000..0dbab848 --- /dev/null +++ b/dsr/dsr/task/regression/regression.py @@ -0,0 +1,352 @@ +import numpy as np +import pandas as pd + +import dsr +from dsr.library import Library +from dsr.functions import create_tokens +from dsr.task.regression.dataset import BenchmarkDataset + + +def make_regression_task(name, function_set, dataset, metric="inv_nrmse", + metric_params=(1.0,), extra_metric_test=None, extra_metric_test_params=(), + reward_noise=0.0, reward_noise_type="r", threshold=1e-12, + normalize_variance=False, protected=False): + """ + Factory function for regression rewards. This includes closures for a + dataset and regression metric (e.g. inverse NRMSE). Also sets regression- + specific metrics to be used by Programs. + + Parameters + ---------- + name : str or None + Name of regression benchmark, if using benchmark dataset. + + function_set : list or None + List of allowable functions. If None, uses function_set according to + benchmark dataset. + + dataset : dict, str, or tuple + If dict: .dataset.BenchmarkDataset kwargs. + If str: filename of dataset. + If tuple: (X, y) data + + metric : str + Name of reward function metric to use. + + metric_params : list + List of metric-specific parameters. + + extra_metric_test : str + Name of extra function metric to use for testing. + + extra_metric_test_params : list + List of metric-specific parameters for extra test metric. + + reward_noise : float + Noise level to use when computing reward. + + reward_noise_type : "y_hat" or "r" + "y_hat" : N(0, reward_noise * y_rms_train) is added to y_hat values. + "r" : N(0, reward_noise) is added to r. + + normalize_variance : bool + If True and reward_noise_type=="r", reward is multiplied by + 1 / sqrt(1 + 12*reward_noise**2) (We assume r is U[0,1]). + + protected : bool + Whether to use protected functions. + + threshold : float + Threshold of NMSE on noiseless data used to determine success. + + Returns + ------- + + task : Task + Dynamically created Task object whose methods contains closures. + """ + + X_test = y_test = y_test_noiseless = None + + # Benchmark dataset config + if isinstance(dataset, dict): + dataset["name"] = name + benchmark = BenchmarkDataset(**dataset) + X_train = benchmark.X_train + y_train = benchmark.y_train + X_test = benchmark.X_test + y_test = benchmark.y_test + y_test_noiseless = benchmark.y_test_noiseless + + # Unless specified, use the benchmark's default function_set + if function_set is None: + function_set = benchmark.function_set + + # Dataset filename + elif isinstance(dataset, str): + df = pd.read_csv(dataset, header=None) # Assuming data file does not have header rows + X_train = df.values[:, :-1] + y_train = df.values[:, -1] + + # sklearn-like (X, y) data + elif isinstance(dataset, tuple): + X_train = dataset[0] + y_train = dataset[1] + + if X_test is None: + X_test = X_train + y_test = y_train + y_test_noiseless = y_test + + if function_set is None: + print("WARNING: Function set not provided. Using default set.") + function_set = ["add", "sub", "mul", "div", "sin", "cos", "exp", "log"] + + # Save time by only computing these once + var_y_test = np.var(y_test) + var_y_test_noiseless = np.var(y_test_noiseless) + + # Define closures for metric + metric, invalid_reward, max_reward = make_regression_metric(metric, y_train, *metric_params) + if extra_metric_test is not None: + print("Setting extra test metric to {}.".format(extra_metric_test)) + metric_test, _, _ = make_regression_metric(extra_metric_test, y_test, *extra_metric_test_params) + assert reward_noise >= 0.0, "Reward noise must be non-negative." + if reward_noise: + assert reward_noise_type in ["y_hat", "r"], "Reward noise type not recognized." + rng = np.random.RandomState(0) + y_rms_train = np.sqrt(np.mean(y_train ** 2)) + if reward_noise_type == "y_hat": + scale = reward_noise * y_rms_train + elif reward_noise_type == "r": + scale = reward_noise + + def reward(p): + + # Compute estimated values + y_hat = p.execute(X_train) + + # For invalid expressions, return invalid_reward + if p.invalid: + return invalid_reward + + ### Observation noise + # For reward_noise_type == "y_hat", success must always be checked to + # ensure success cases aren't overlooked due to noise. If successful, + # return max_reward. + if reward_noise and reward_noise_type == "y_hat": + if p.evaluate.get("success"): + return max_reward + y_hat += rng.normal(loc=0, scale=scale, size=y_hat.shape) + + # Compute metric + r = metric(y_train, y_hat) + + ### Direct reward noise + # For reward_noise_type == "r", success can for ~max_reward metrics be + # confirmed before adding noise. If successful, must return np.inf to + # avoid overlooking success cases. + if reward_noise and reward_noise_type == "r": + if r >= max_reward - 1e-5 and p.evaluate.get("success"): + return np.inf + r += rng.normal(loc=0, scale=scale) + if normalize_variance: + r /= np.sqrt(1 + 12*scale**2) + + return r + + + def evaluate(p): + + # Compute predictions on test data + y_hat = p.execute(X_test) + if p.invalid: + nmse_test = None + nmse_test_noiseless = None + success = False + + else: + # NMSE on test data (used to report final error) + nmse_test = np.mean((y_test - y_hat)**2) / var_y_test + + # NMSE on noiseless test data (used to determine recovery) + nmse_test_noiseless = np.mean((y_test_noiseless - y_hat)**2) / var_y_test_noiseless + + # Success is defined by NMSE on noiseless test data below a threshold + success = nmse_test_noiseless < threshold + + info = { + "nmse_test" : nmse_test, + "nmse_test_noiseless" : nmse_test_noiseless, + "success" : success + } + + if extra_metric_test is not None: + if p.invalid: + m_test = None + m_test_noiseless = None + else: + m_test = metric_test(y_test, y_hat) + m_test_noiseless = metric_test(y_test_noiseless, y_hat) + + info.update( + { + extra_metric_test : m_test, + extra_metric_test + '_noiseless' : m_test_noiseless + } + ) + + return info + + tokens = create_tokens(n_input_var=X_train.shape[1], + function_set=function_set, + protected=protected) + library = Library(tokens) + + stochastic = reward_noise > 0.0 + + extra_info = {} + + task = dsr.task.Task(reward_function=reward, + evaluate=evaluate, + library=library, + stochastic=stochastic, + extra_info=extra_info) + + return task + + +def make_regression_metric(name, y_train, *args): + """ + Factory function for a regression metric. This includes a closures for + metric parameters and the variance of the training data. + + Parameters + ---------- + + name : str + Name of metric. See all_metrics for supported metrics. + + args : args + Metric-specific parameters + + Returns + ------- + + metric : function + Regression metric mapping true and estimated values to a scalar. + + invalid_reward: float or None + Reward value to use for invalid expression. If None, the training + algorithm must handle it, e.g. by rejecting the sample. + + max_reward: float + Maximum possible reward under this metric. + """ + + var_y = np.var(y_train) + + all_metrics = { + + # Negative mean squared error + # Range: [-inf, 0] + # Value = -var(y) when y_hat == mean(y) + "neg_mse" : (lambda y, y_hat : -np.mean((y - y_hat)**2), + 0), + + # Negative root mean squared error + # Range: [-inf, 0] + # Value = -sqrt(var(y)) when y_hat == mean(y) + "neg_rmse" : (lambda y, y_hat : -np.sqrt(np.mean((y - y_hat)**2)), + 0), + + # Negative normalized mean squared error + # Range: [-inf, 0] + # Value = -1 when y_hat == mean(y) + "neg_nmse" : (lambda y, y_hat : -np.mean((y - y_hat)**2)/var_y, + 0), + + # Negative normalized root mean squared error + # Range: [-inf, 0] + # Value = -1 when y_hat == mean(y) + "neg_nrmse" : (lambda y, y_hat : -np.sqrt(np.mean((y - y_hat)**2)/var_y), + 0), + + # (Protected) negative log mean squared error + # Range: [-inf, 0] + # Value = -log(1 + var(y)) when y_hat == mean(y) + "neglog_mse" : (lambda y, y_hat : -np.log(1 + np.mean((y - y_hat)**2)), + 0), + + # (Protected) inverse mean squared error + # Range: [0, 1] + # Value = 1/(1 + args[0]*var(y)) when y_hat == mean(y) + "inv_mse" : (lambda y, y_hat : 1/(1 + args[0]*np.mean((y - y_hat)**2)), + 1), + + # (Protected) inverse normalized mean squared error + # Range: [0, 1] + # Value = 1/(1 + args[0]) when y_hat == mean(y) + "inv_nmse" : (lambda y, y_hat : 1/(1 + args[0]*np.mean((y - y_hat)**2)/var_y), + 1), + + # (Protected) inverse normalized root mean squared error + # Range: [0, 1] + # Value = 1/(1 + args[0]) when y_hat == mean(y) + "inv_nrmse" : (lambda y, y_hat : 1/(1 + args[0]*np.sqrt(np.mean((y - y_hat)**2)/var_y)), + 1), + + # Fraction of predicted points within p0*abs(y) + p1 band of the true value + # Range: [0, 1] + "fraction" : (lambda y, y_hat : np.mean(abs(y - y_hat) < args[0]*abs(y) + args[1]), + 2), + + # Pearson correlation coefficient + # Range: [0, 1] + "pearson" : (lambda y, y_hat : scipy.stats.pearsonr(y, y_hat)[0], + 0), + + # Spearman correlation coefficient + # Range: [0, 1] + "spearman" : (lambda y, y_hat : scipy.stats.spearmanr(y, y_hat)[0], + 0) + } + + assert name in all_metrics, "Unrecognized reward function name." + assert len(args) == all_metrics[name][1], "For {}, expected {} reward function parameters; received {}.".format(name,all_metrics[name][1], len(args)) + metric = all_metrics[name][0] + + # For negative MSE-based rewards, invalid reward is the value of the reward function when y_hat = mean(y) + # For inverse MSE-based rewards, invalid reward is 0.0 + # For non-MSE-based rewards, invalid reward is the minimum value of the reward function's range + all_invalid_rewards = { + "neg_mse" : -var_y, + "neg_rmse" : -np.sqrt(var_y), + "neg_nmse" : -1.0, + "neg_nrmse" : -1.0, + "neglog_mse" : -np.log(1 + var_y), + "inv_mse" : 0.0, #1/(1 + args[0]*var_y), + "inv_nmse" : 0.0, #1/(1 + args[0]), + "inv_nrmse" : 0.0, #1/(1 + args[0]), + "fraction" : 0.0, + "pearson" : 0.0, + "spearman" : 0.0 + } + invalid_reward = all_invalid_rewards[name] + + all_max_rewards = { + "neg_mse" : 0.0, + "neg_rmse" : 0.0, + "neg_nmse" : 0.0, + "neg_nrmse" : 0.0, + "neglog_mse" : 0.0, + "inv_mse" : 1.0, + "inv_nmse" : 1.0, + "inv_nrmse" : 1.0, + "fraction" : 1.0, + "pearson" : 1.0, + "spearman" : 1.0 + } + max_reward = all_max_rewards[name] + + return metric, invalid_reward, max_reward diff --git a/dsr/dsr/task/regression/sklearn.py b/dsr/dsr/task/regression/sklearn.py new file mode 100644 index 00000000..c3777a30 --- /dev/null +++ b/dsr/dsr/task/regression/sklearn.py @@ -0,0 +1,35 @@ +from copy import deepcopy + +from sklearn.base import BaseEstimator, RegressorMixin +from sklearn.utils.validation import check_is_fitted + +from dsr import DeepSymbolicOptimizer + + +class DeepSymbolicRegressor(DeepSymbolicOptimizer, + BaseEstimator, RegressorMixin): + """ + Sklearn interface for deep symbolic regression. + """ + + def __init__(self, config=None): + DeepSymbolicOptimizer.__init__(self, config) + + def fit(self, X, y): + + # Update the Task + config = deepcopy(self.config) + config["task"]["task_type"] = "regression" + config["task"]["dataset"] = (X, y) + self.update_config(config) + + train_result = self.train() + self.program_ = train_result["program"] + + return self + + def predict(self, X): + + check_is_fitted(self, "program_") + + return self.program_.execute(X) diff --git a/dsr/dsr/task/regression/test_sklearn.py b/dsr/dsr/task/regression/test_sklearn.py new file mode 100644 index 00000000..193bf6c9 --- /dev/null +++ b/dsr/dsr/task/regression/test_sklearn.py @@ -0,0 +1,24 @@ +"""Tests for sklearn interface.""" + +import pytest +import numpy as np + +from dsr import DeepSymbolicRegressor +from dsr.test.generate_test_data import CONFIG_TRAINING_OVERRIDE + + +@pytest.fixture +def model(): + return DeepSymbolicRegressor("config.json") + + +def test_task(model): + """Test regression for various configs.""" + + # Generate some data + np.random.seed(0) + X = np.random.random(size=(10, 3)) + y = np.random.random(size=(10,)) + + model.config_training.update(CONFIG_TRAINING_OVERRIDE) + model.fit(X, y) diff --git a/dsr/dsr/task/task.py b/dsr/dsr/task/task.py new file mode 100644 index 00000000..8574cb08 --- /dev/null +++ b/dsr/dsr/task/task.py @@ -0,0 +1,86 @@ +"""Factory functions for generating symbolic search tasks.""" + +from dataclasses import dataclass +from typing import Callable, List, Dict, Any + +from dsr.task.regression.regression import make_regression_task +from dsr.program import Program +from dsr.library import Library + + +@dataclass(frozen=True) +class Task: + """ + Data object specifying a symbolic search task. + + Attributes + ---------- + reward_function : function + Reward function mapping program.Program object to scalar. Includes + test argument for train vs test evaluation. + + eval_function : function + Evaluation function mapping program.Program object to a dict of task- + specific evaluation metrics (primitives). Special optional key "success" + is used for determining early stopping during training. + + library : Library + Library of Tokens. + + stochastic : bool + Whether the reward function of the task is stochastic. + + extra_info : dict + Extra task-specific info, e.g. reference to symbolic policies for + control task. + """ + + reward_function: Callable[[Program], float] + evaluate: Callable[[Program], float] + library: Library + stochastic: bool + extra_info: Dict[str, Any] + + +def make_task(task_type, **config_task): + """ + Factory function for Task object. + + Parameters + ---------- + + task_type : str + Type of task: + "regression" : Symbolic regression task. + + config_task : kwargs + Task-specific arguments. See specifications of task_dict. Special key + "name" is required, which defines the benchmark (i.e. dataset for + regression). + + Returns + ------- + + task : Task + Task object. + """ + + # Dictionary from task name to task factory function + task_dict = { + "regression" : make_regression_task, + } + + task = task_dict[task_type](**config_task) + return task + + +def set_task(config_task): + """Helper function to make set the Program class Task and execute function + from task config.""" + + # Use of protected functions is the same for all tasks, so it's handled separately + protected = config_task["protected"] if "protected" in config_task else False + + Program.set_execute(protected) + task = make_task(**config_task) + Program.set_task(task) diff --git a/dsr/dsr/test/__init__.py b/dsr/dsr/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dsr/dsr/test/data/test_model.data-00000-of-00001 b/dsr/dsr/test/data/test_model.data-00000-of-00001 new file mode 100644 index 0000000000000000000000000000000000000000..ceaaf9baa491c86564b8113578cdfdd170170eec GIT binary patch literal 83444 zcmc#)i91x^8@F%SvSq962{R)xGxt3V38_>PlC&X7rL<7CWDg-_DZ2^@sbuaw*G#4L zORGvMNm5Y~(kAiqU;OU#+~+>`y=QyQ{hV_?@8`Vd`pZ=)0)GI5Csh~?8xsXhaiF{H zCY;uri;No<0^OKFceYnI+m=S4LYp0EZ9`6Tpj0*D`F^IGjce)4&vrMT_gQC`Ds<)w z)jc>4r*k=-Ih`Ck?e!dPQ5E}q^;-6y{1T4U9T9?Mk}A9|et{R*Y9^G9wG_C?H8#Jw zn!$_JI%}7)wB4@nORezHY!)xiYPsOKZm&Snv%z+MPraS%jRI6ZWR9a1GKJ>Hn*|r6 zB826W%Z2poxMr`Xl{_Vl%e+%Lqk_v(i|ppK(7cC#6M0SA);xQOW5P3k{RB~a6YPcv zp^#S>%+qza$y>tP-)ufN>r#n4 z?Y-lIw-%RpJO4W-v^X~|(EO~-3mWkiSoiz!Zazrisk?LSRIR*(j8$F&;Cl*1$yauK zjlDwOPk)P=vEo)Pk@3WPHsC<*57*d~0kyOKu^oe}oj zePU;@PfHLowos^&BF$rHCJ5dxOyh|)TH2}IWDC9D)d&u+;G(Ptl|nB$G2Xu=t85*z zvGC(%TDYS!T^OvcEgVi)7wnF_Wj7G;-Oe*H#jY(QnPy9XmeX!l?}Ev#B0A;_D`5J+cK3cnBWgh%IIw$rsA zv$IDh>|9D|o+h0tn4MKGTp?@2s|s-D4dvGgrQ57|H_S7IvNJPzL0b!i+fROJ4jfSC z6_gkW0$~l$?01@->a%p7IVZ(7(Z5tU)U=8>*m9Lu`m<3eSy&|W6y3~weCQ;P#!*6e zM)Q6S?GT=kY~qz}lD3=VYS|fF8y9@D6bk)KQiPq}&AiRc#?)7X;Q(?XXCUh`V7Dyrid{EHlh?bg zh3DVaB9ysh$(uEGf_JMgljryAoxrZGoTnUc(9YK+l=mW*;Mvv^c1p2P!f1OC{QY3Y zGrlS-OkXK0)D;-;>b)_~Q6*7mVVB2qGvy1b4&4^0JlM;Nk5m`lP&;YIme)7DIICR(&P2T$ga^);tCyq)X#$7|F1&U!iIk-Aw#?gcj2Ge?W` zYhth+oKm=-hZ^{P8e2IYp^G?wzr=H!HmOn(9q0JP_FCMlVJkQm+SQ~W@-0U*P=kN^ zKpp>!TrT-zxj1q8=sxZePj7O~c1ym0bU%0SktiAQrh-HJe&A%ve&)96hEuaVCAe{} zU3~XLflUYRL~*yLNl6uV+T;17@?7IHzq`a&sH~Hl*{@eZ&{Ng>0$Ba-iAUrR`uUF^%h2CnRcFa;{=yqxjc?< z9kzfQy;p(X*dM^=PORnR+3Rp)E#gS$MUI^DNB8+H9h!X286jNlALsZE$4l6+p6uo< z3vuVlpRObStQTVki5=p&T-WAwzt$&xl1R?!UENnVZn5F>^CZbdTUEFke>AyWhZM;9 z^$dPfJSi()-#-f_t5&80>64$??kdra5m1NH^;&d%Yv^1d?WdH2A6R*-a5+RUasJd?qX4&)C>6;Z|1VKqW(5{SPfnczhKt%FQ%ob z)5nwj_=`&uc|@LD=MqfL?(yIXj;`d~79A%YU;pK3$dvLq@x%Orp>L$-+rz{Tt{J&6 zR*oyOgtD=c&L?fdLO9#KjQG4P7rvo?4%yBzBpO^Jxt<67$jZ%Y_;=3!;POD69I-jW z`84^Rv+=(l+%c&ml#%%yZcgMZ(!ca^Q&W2-S9|A6{+$cqoM*#%oYhauxkqwMDBlAd zGIUCXYe@`ROQKxz@W@F{Y~2!mY2JE%g~dtIX~c-QEg8<~p}tsKP621!#Djk;Vx~>r z@k(}7NPLs%*VHEWVOxHMXeUce{0M7sXge$3ZI&fF#h!I4u8m>Px0D$jt7++Tr_9ov z<;N1!{KVWV=wo)|R<|yln3t_pXZIBsaJHbRx&olSqO4ggP*UVWJ&n+llNu~_h$|_L% z#`@K)ZP^!{!|Ktv&YJ1G%u=B(nKl2f9J7o*!F;`CrKNt_ZtA z%2pQ7R8uVWZBt+_c%97j6bom`wJWmHXL++O6|}R09;>jntj=N_w9;U148P85k=$n) zuknKUFu;uIYpcWT(mcjAlWAwHFuQHBqmi;O+H{+F^7%nl?W1wlrv*tYp=K`YfVHFL z+LN)YOIg1e?@XeY+MRZm6}3+-f2Ut(eQsv3PHm54O%%Ok)jpfY`r=r|i2A6`EID+a z_02in^5NhN=CbWM3{@V3v44I7lXat(F^C1`M=R7B^SDjSVeMw7Sx7%)xmqs6_Rk51 z`IBNM6vr?u`Zt)5&6%<&ICz&S-Z#N)eqzUrD~M)RtXarB*f*DT90f2RFY~r&R^HE0 za^bOlon6oB5LIEaYenTvdnSuN37-DftFuSb+HEH99WlG0W3LFCCe4ocC4@0?=Whq zB}|7Oa+WU+)LG8nHPdpTUmgpOH?erF%PdRnkFq>Aer04DWiXGp+gbYjdu_RTMn22- ztv<{5mksOmj|Z&07BN@LyES9-4t6oEe^;1N^a!(gyHEv2}WS$>p1cpCi z{rJApGBV^i%j=;WYmH(G>w{k}tL4vZ)?KMm#^M8GjMT$rtj6zQmh+TnGJU>^S@>)c zFpxqG<3?qNh0h6R3u2R!#SynYM)u<_=K2c@&8^WV6s-5F%}yCYVpA?(p{0*y;%sNViIwszWz${i-*pdp9Lyey5b8yT#FX&djV1xeeuG9y=Y-(BV0|{gZ@ibK?O^ckpIMN`gTkr8df$&Vzbof zr3D^nna@#rTvm-%yXa4!ciDq%tMe&28AZBTeJ>hor19~k`dESQOE-RYh4Agi(1FaQ zv>JZ|XEw~H<@YGluUl2omVaC5g+WnB*&_*Y5Ao=Gei_v9=p3}t-VD+5+4MYbKe|H0 z9`&BFgJ1PLlu~?*(wh-Ql&^gWoaV)7^qCB{pKpjlmCVra8+H0*4@tCTD4|yK z3O)C>Ju0~M6i+FvK(StFu((+QH%1(T&ujJJ(bFPa`Ex#+88HF8?>=bH2YvK=$ybae zb|Le!5;(!!iZ(pDKwQcD4XDhRUVDB4VkqAL(ab`yvhILAH#Z^?&Jj4U#1y$`EI>b= z{>J{T&M-I39gS`USjc1mdJ$tQmQ&$_FP3k;ePmuw&x z{Nj>k#@}q3T~_b~3HSK>mzCL!DzD|E)j#>SWj3YU z6(_funs78;8j@>l-TCWmg81vGjeO+2gI|-}#1D*CC*$|-&Nx%pxj=A)k7US-ZH~B2Tx@!e-|>R$a&v#)0%t%v#*akhc;rn)zlHUrz|xe=7{w zuA7587FRf@~zN-Jpyz6Dps9Y+|RJm!uEZJmY zA%By>2oc_8OuoKpAt`)hUL$#v`Px*O;U#{8Ig(fm5kv{;DTM(%>rHgzeCDHd}A$A_U4Y#|N zqn?opxSrdBCwWicN*|B*-u4w9@B0H!_ZMNu*dp$^=a(RraULhSKH}GxcM`#J(^T_o zFgN^4H!gqL3v8}2{Mf-k^*Lr zN?PIVT4bI4gm_*gPM4kyK~DX%Xs>-{5UF8KOLg(^j?X(`;Qn%Y@RB&;)v*p#zR1v< zLw|xadkY?yip7iM_o4}bDE(hj4c?SH3k@K7RJlEnE)bky&)#E?oPrWyc6>5C*WlvU z$0vzHj~>GUKLb>P8LFu9up0*8^92@n{k{Flq$RZP9dIgaq!`l!VXeJ%x`U zYmkqj6qc{T_;lZMc%L`t)+vB)(Q)iPkJG z;V#%}focmppz-iEeDhBo=q?aY_SqEHy151Y3f4rIR@kC<^<5x!TaVs78i1B1T!QQY zceG$ApJ;k=3TG;r(bke)D1og2btUuZ^B>N@8CheL=#YoWv>LEFHV0x;HIa%(C+Tve zfa2a(q0{&$aLrFPV&2Flxz$!kro02s-ROvGL!UvI_*UE*%fvcY0+G#IeKaJt92qD6 z2TYea^eAsPa%|QFW9^Md3~$6=FOA@q?+fYQOP!EN^A!kv{E+%9@(LzsU36y57HcZK zg!@l7gQMJHI&@S5&rdNxUg`rlX!jE!({xdsJjMA~s)q2;JiIvQEyQZ&gT7i8Rv8w> z6+9bcCnkf4w{FvGxFg#CTAC&|1S9S6eyCDkhjuym5tZt3#G^nt`ngLgs?9S*8Pp+M zu{Q)IZQ4mos!mWluW;$koK4Vhe>1>$Nm@K?A1cd}NB@Od(93!}>Fpb@Lej`m^g`DS zugiC%FE4kceJVP!1MVm6PNdUdunJ9R#Gxbl2WXG95xnb1Cc5#Li}IEqrp-$m@xcB{ zB0ResT)(YBv3aG$(Ycu1B^>igW|s zgdUZTfL4ST`g83H<;^K26gh*~+M^rnG-spsN++VhXdH%TPUBJ97Pi?JQ&a0wC_5`% z+!>XD?|;`tE2XP&f4nUEYPJz)-uMkG-!4WTkDIv>9}=Nn(GCTf&7#XoZh=F62Q-uc z5-Dtl1j}sj{}+IXKtVu7nynWDcN0qu$UfjgF~qSo=R&?@pB*0!jE2je@uvYLzX zGZY~8#S<8+834&M+Vt!}b7Htd42j1k5}tlMys=IMRXzPn+@=7{njZ`b`p(!j=ssMX zR|=bkk}z(XgGPGiqV+6GbmHJMIBA$jX&Eg=?GnN8=8HXYc!9xb4~BOYHuULiPjvK{ zBXY8ShZlwO&`j<&x*|f1F5z?O2OmsPVs1Xy;@<#%tHnY?j(2gFcpF3{yu{0P4seCU z0mK%H((+XySnRYbJ#cFT&$;jr6t70og3_hPO>Ofuh0~@7!Vg1<>_KF$HWLk1Y^6nW z81#_peB@Z`41K>vP)HiR5A^d-Q@BOclW0wo*9B)qr1TjnLpRU6fQ~gQAvC!fCN` zDsY(zn*V1x^vM|@o=qgkt;_+XQZu?PW(p2lYk@SXhqKZ8l<9)gkR)aZZxof$ZfQ}} zu4PEy+WwhZSMrm{ev<*=DxvtoxDJXJ%LP^D4LBX}6b8TzSoeJ)TT2@vYxcl}?zxDs z&Gdk{u8iJ~s98oIy7ohjv@(`8U1g39)0=YV!CwiOqAzjfo`&P(i6?G=+>>3w9kkn(!Deb{N7x` z&F3A_)Td4K?BXD5C~z)v7C%JWyjX!g&o@UuW2>k?zfF-(W(|xq4@2Lv&(JKXjt1XF z($5pzY3nCj5YzDl<=nprOPVeK&m|^k%%~8&KWKr?m3Q#!ksj(TnTs4Yk77&HZpz{i z8@Y2#&}Soxm;GKrYNGu%(XjN+B3VZA)+Ha<(V@oT7_(VJMXONf75?ZAO&Y>~>9 zO~l>BGT_)0MraKE28jd})H=t2wClix#q&l0rv6 z$snDQG|+!4Njtpx0t;*wpumSykfv0Rt5!$iu|@Uxf`UAVR;PgG{$P zkbK-g#LS*%%<&E|w(2rmpHxC)!Z2!~uLJm;mqkU+Tfoxl29e)hC`11P z^mvU?wNt57YtIZI+j0rP3r|RCC5gln({(f*09DO(M31a8^7Bf8*m=IF*|HBpC+dMy zPzWB2txUdM8(+$*1S>!UGqH_IvRlt|Ug=Tr}z=UVi08Q#_^{O6*bMQH;eUP~oow zUyNl?o~9B=UtWoYXLV6;R}#n%IHG_RC!lS!BV^0Ig!RsE!6sZ8tr<~Af>v?l8s>n8 zdM-nBz7sWi_!}g;l)*JO0idqGAj%U%|E#KEUaAbeUPS{@Noj<}A6=voYlKewN#U@i za_HCiJS66H3;Kl}q`}fw*ys0>_&u>4-m{UA`SmZHm^~jEv{g{HED3aD&UH9l zSjsJico@CIBBBnY!INX3sooRCB-M>Pwl}@t*?yJ0%2Z`OgAoj~abii9lNG z&SAMaif}Zmp@u(hBi7IBg{wXKwBPCbSW$;VzCEc;zhBh~;TI%nJ?S@u2pNfJr&;u7 zt`s*!Cl0MzJq*JVW_0MahtqS13hGL2rdDL9!2>BDsz+51Jnr6uu7Ut4&0m7~w$=bAeTn^^iR19ozoB%NE46w2b<@oc-SFnX0A8OM519{!iIPqJ;Hh#PZ%(b}{z=Wm zZDNPu@p?bp`OFwGSFHfK&DGed3ZaO8Jt%M+0Md7qxbA0*(60h~G?1Vjyl+oqj0TM} zwjkwbN*}oJ0NegLz-5SAqg$u{qslTcbv-Wx`L_8Ib7b17;FWdoq0fqrwbY=uzsV=s zhWPmV_C;9g<2d>3=5?%@;7TjGo6_q{H0e-}xwPBiy?A?135tlD?-%e4RXQ4e>va`72oE!5F-1RYGCr0UQ1U>CbBG((5LCq7=pKfFdLeg9pk zd&w`bUd^UG_XT2~)JU}Fvj)1Gmq3ra5uqcUJE@0(+Gy(8e{jJ5ICN|M!slj7;)VOo zkf|_$xWoGjf;BHlU6IwCgfsH^Pgf*ngh|rei|66xUU_u#ZZ3>Oo1t}Eo$#Z)7lhsG zTu|Jx5Z8MdL-frrkd>l~{sk!_*Q7t-R8xiJrv04bdNt53VukLfbQAaP_u{Q9%dv{3 z1ZqF1h7R@_(Uf@+cF5O3`@Y`8o_-B*km|xOB$rUcv|s!9%u8(IV?(E>+y<%Znn>$- zKkQjzjutf=(RtlBxXy>FAz|_-p5f#OBde5ABU=}-&dx$588Z=kh9TXE43Yf5TOiwL zhU5#@<6&h@+WEr=tZ(Ov4qMuzBlG=G!~RZqmg$F#u?jjV>x#bxwSiK-9a<9+4s$0D zq288ASm7IpZi|It`z6~b&3~5YRfY%B9nYoxOCtoI8x|%)o6aG0vnrMDS4pC+*Q>x2Wke?g z4nY3LBlL^WP(+~bKpTsoGD|yDp_qrp1nG45@nbX*e3W*YSO8nhQ)rR@^EEWJ4aG}m z!pvz5xZ0kBh*)Km>Hm;04BLr=_OZb1o(Y<>|2Xk+crkjhBN-QO*Fo1ab76(IA2w1n z!K*~nk;u?_aM>qE7nN>>PVr30e@39y?oCj?Tnt?hZ=?DjEe9C~gkJS7#bT+Q+z|1{ zpy^wU?>g!uxsVG)>?{Fs4y!@*S2d(`as_%a`2@rj_0Y(doy5cI3+O1WBfN4oL_?h! z#OK$iuzc5jaB}*D*`zA^K|TesS2WNG3sJ_bCSv8fG)xLiz;1f}cwDW83~Z7iJ0TP+ zhaEz-$N5C092*6?^h5AF3G^lY9yS@ugYFk9$h@%u%haVHiOD$nyrC=Yn(9Vp-u415 z@d9Y*m`ghk2P3rsb!v0&3OE$A2Y-_^pvOZup%-h}bVmJh`gr~q{P(doePxJ^?j0(| z+@m~H)wULGnVU^D7D$4%Z6+!?s{-+Fw$h1q_H?PL6*`*i3fa#s(4pvlg1M)Ka_T4q z&MZ5`>k~j4sgBw#a-eHhKJI^M4j;lPV1HVU6DI!RqIPj4lPE^-5-Ot%Rt-dUsNny8 z&q4B*vSjq)dYr$>5jm{-4!4@EP-tBV*j>%UvO)Wiq+LE_`z}Iruf73s^%hWg)Pa4Q zbP&7mESKZ91?Dh+6KBNeSXo;zUGX0o-fB&4$ugtG4=bZ> zWh!*J76U1-JP3>RW1-+Kf!x-{;Vn@ucw(~&GRRgzcjZk{i_9rlI&UVtU$GncdDpBUW5dXUzbe&%i<+KX4l4;;Sc^oLF2obU)j?6*_!E}xl zF3VYmPpF&1;I~0=CLe?L%Tk0Zc!ZUUSxkH-S~ES@1^55TR2BQ?2HYZc%tm9qg5rrkS@f390Kr)->TR=wH{weuLAVz zB4L}dnP5H0hVlQlLdoD_Ft%R>g@IMzd3Xi3-;)9BvTH%^{aN@Fx*B)=Ou(0vep93V z#c+Q{5;r;FA2GAG5>A$#1QYcN@_uA46q>5y`1vjnD&>Xmc{{_@iG!GB-9%V_%L3C= z^NH$6USP;Sk`&3}*KLd_Hq3#f!;Y(?-l~Mz6l)g*4wa0)Io1}t+ScJ%biePv( z2ENreK%qho?ui)YA5-&#&FM5TdiVo3jnz$z`FtRruDeaC&s)JAt0|-I%@7b7OCNJq z7rr5e#rud?ZLgb_$8YCL`SlXvwPA{_J4&=VjuPvqpJ}0^RNl@};>xEN#5u+&CA({sVo#^Pw|$h7SUXCoTaHlj z)8CV)ZQUzIiEGPysQ9=~)Y-sMLS1*1(o`BD@@I`w@n(HQ>V{E5#buNjavCK9*Nzgg z{V$02*u#WP)(67(>leb!@HJJ*)FO-|u2CzqpAvjd3P0`eC&FI*GqEa3gkNg2i3l&)b?4Qsd0%{#JaPZgf;z+O8(tPoc-@Q5x3wIr|H;oQaJ;W4WD0+_? z>i*5?yw^wN{eDQzw7W+<-4Q~B=zJxbC$AEhYd;c3-v$Uxy_;0i*o)c$KH8cqoZvP?xP&+*`Y~Q>%`&I z_3AiW7E^EYRdD~(yZj*gb#S}kJrU4q0;}QxyX(bsbQAW&R`$l}F=GlmKK-wh@!Q*l;#A`W>Iip$%g@cZ4#goW%9Y#x}5 zZ%FOLA1|DvYS(Rn@csbYHq=W6F3P~=^Z3*`ZUD%EH#W9f&y8g!!FT1Iko9c~cs|^T zcVDr`iGvx?mX?5vVKaEUC*ugaqgeevC2A$-7@WPEfg>A^$3uY5K+gKHI}qNTLhtd(3p08=}s9=n9?;C^Z{LSgiIJSaQ~zjnug=9~tYTGs$RFD^j$qI$SsQU6|7?5!J0#rEF zgHyx>JT6iTWp$U}z{v|ZzUdO&YOjM9wR$MBtiVgBbr7Pf4$B|Agk9kxJY04GKR3FF zi>LJ+TdEF@oxcc$wj)&8xqPf0U4>8U#N#Ei>%siNMZEHSEi|sZh#Tr^@n2yb^s0WL zicBx!k%&tmp;Qa!jW5BFZ#|y`|VAJGL|m%WEF9G{hG1viy!#p?{54~;TN=}G=i-813V@_j3<>R!1WIsUzGif z9p3zeHR?k+TU3G`%zBSmNe8gO*SCc5iX&}z!=SbM4f z=N7+%fsOKX&GXZkS+799^f^gH_(-EX6)E&lM1k(ec?8$eWKeXp3~jt#4yB6s;ZQ>f zTHq~CXP#3)&-BKKAZK|Lyhs{pSV$w2^dDGx^-DZ?HVHz<*f_OOhVI`YIjti$g49zP z^moled_Rs0Y3D_-jGg=~CvqXcbz_y_g)zmLEO`UK2p^PqSBP5g)T z9rw%2&|%}E=x;m*{hyQA<=rSAd?1E8f~;|`wk+MTRTd4WjbopAGW6ibDg3Kl5*3_# zfY<8Jpm+32(Q2tbVS%AE8hb55+vUo@g4DkdI5-G*_21(*D>+o;{R?~ElBJU#4dFlE zC&5iw6#e$-CN{qw!N^LEPCFz=`{_*K7gJ&=zfY2O$1`Zpd@1zcoeWJn%Fy*yGH9!+ zIQ^_K35){LDRGAZDp*35Qc>kj^Hv^JQN#hWJ8NOvkbp=qj>fMV;;0)aG5zR-kuVF}p!&Uskzy#eA@orP((QX`y(AdiVuBx@%Vmm8MZi-jw?iSfTwGS*~3S`@jrV?B6b%HZ4Cx* zi74n@6bB;JzQClm;G#8s)O(X;sy{4^{fOQJdU3m9!-WKFQhF3Zo^9e(RX^Yhx39-? zmdT{5M-FV14Z&r_(O{$D3jVG0AnQsRB<+ub85ewU@c2%6vh68(HJAxvk|)5O_liETBTW0OUo=M(#-sWR)b=xjX?>Pe1_0zn!^oTUEgvM`KLVFUyTPqSgZ}Tb z9*WD`&-b??MBy#JzjnhSSx>e>^ z;uOPL_-IMc?^B1VMAeJLkk@fyqs0PTw<(%<8#RnY)4SomSt7jn`kr_vHv(4e5_F!u z3|9Di4%^j4;9a)Sc>B@UkoD|5B+c)F^7Xr+I>LZ%c{2ye9X&~f@kLSVB_P>G*Ky&B zacoui3cN=<@ZB+GdWPIReE5tbJQ|zAK2s04vhP>ml2tiz`HQ z(u!%{`c0F5zb~{&4dad9x^T_gmqe6&F}Bapp?@YSp!4i!pdVHR%)`EfY^*%}MpOdX zU7ttjy?F(*+Ad%zGYk5`rTf@8J%G66ZA|T$!zC&^b`$p_mr$EGz9+Y&&!IeyIue0k zPI?!66IV_EaW`)^aZl~B4f}Nnaa`&HS(M^QeR#nmPNrSv9{(Ut7|F1S>LCSU=gv)p zc@2l?PYok%UyhQAk7cO^7q$~&-JRrE@CIT>(rU`f&7Cq^&LVF2#Sp9~G1Rh_80zA; z7~*7e3{ieDhPV|KLxk~T2=})!)Q;X5D))R00heQ_jkBT&lP@vUhyEC<`(+IE5XVrf zN@FPh))?yS?ddY|V~EU8G1QmmF~pazF~o;uF@*j77^3A<4B=-ULtTCzL*1CJ)2Ka$ z8hsZ-teDzEEZ*%$RX4393SS-|Zl2ykt+I2VN+KL6%EE`x(5&XZ-4#SgrMeU6K3NjF z#;TP0m8?lrbLCm>oO4-c4 zM$#Ae6Ad#psK;m*VShf9s$9B(nsvpT2wWXbym@0ylsSbG%Qx>Oo+a}r+4sSeK~V&i zQnQ?z`%s*F_hJa~-sb>ucc&wjWU!IA?6Hs7G=oR=c|}lH6$7c~UwX-hz8k1YT~DHx zyPG;{xRYr5yO&rWe~@4^0|@6xEyB*ui+Fu3fI8y7m)h^Yhl;qqpL#rBhT@D6Xu+%3 zFnjey&|lpE<}v}ub~-L**03QOu>n+?bsFc|IY7a<2Vy6!12Ow#NU<_S{~ZLJ?i7Lo zrKOO=&CeWnx#{umlL@}9EAYgyCYrQ$L_^#CP={9^%o{Mmoie-8(I^Z*bi&cOC0d9M zQb0vBr}b&IBy_NsK%!f8=)c2jkW5GhZR$0XepRuBj@fn)ss2r+e9hEpt3LDNJaXD(jxTraT=EWz5;b^&H|b3F=W7`9FX7Sp}>1L4q}*~ zuIedBdF6-5Fbm{7YX;5f+=g7TTETW}2+HM^61INN;mRR>dOyb*MP{iWW^+3j-aHQ@ zB7x|j+hurZAdCLZ1LW%5fxAO2K}9YQE#I94p8k@k{fRhwE-FUhxjWJDCIz&theKqX zsv*V{(m?A=B6Y~{3XBUDqNd@sNGJCz++7w!{YmvhPA|?v4(%V5)v$AV&p+plA~imm05QMHA;qAIT6Ee53{I88Bk5}LvA;2A{)Ov6 zJg%YiBHw`enV$sOE(05n9Dv=IY$2fX4H4&f8+Lgm!66?})SK-F3LBcKwbT2JbuSMS zihHEc<8&p^E3$&DYAam3!wzMCZX(nYFH@gpg`r(nUFp0Rw}`F(&8NfNtdZ&T`IInc zYq}|C7HV-jM#xbagia=pZa=dg?>AVD&agAF7DJyN>YhQj6>_J!+itY8#hgyz`~!Cf zKNOHAM#uYeX~$`-Ub}N0!Wls{uhS5VeG^Bw_sb!vxFHaU8|N1@D5|)Ff-mVGz$#1| zw^v0&b$%`AD1M;SSo2_jRS3P0MkwVmc`BS7029Av>ZV6K+;K(3_0$n!{MAwjEkf}1 z_9xBjr1ZRb7I+$9X1?}Q`O5`;_{iP<*?NcX>-m$pp7X)B7ah2LX_6t?1OxtFP_{c6ZLK9KOLQbgJ%OW^q> zJFt0TjoGiRLfp^kQ@rCOjiQvO+#N1;JV4XDU@1pa6Gz90L&IBPMEIEs-TQ}Jru!aNhCZRrJmy1B?m6xY#T!Jc z!fXi4QG_)!&l7ayD;T}%1KIg9sODM_bf0v z6d#Tek_6j8kDL88A6}~UfmsJ1qI+gQbZ>9Oma33@xem+ z9z`B{35uCBpl9JnVr_^Mv>exigZs}A6Q+FNHSB?$!#|;>#stO(Ur-D470^c2UBqCZ z1p4hc6V@k~!=S7IR@>NsZ#zm+4=19yt@CqnR#gWUUFJYE)`nyC(POwRKbI(|)5P-+ zO=}iYQp6ZF&T-MDaGYihp?t3u&%91jo)T@;xu)gVg`ti6r~lX46_@bJy}Ph$*$@1= z)fx}Xk^*fnIa+FI9HrhSLT{Jqpvr@0W0CI&c70^i4rm*tFC9a;{t2f;jFuy;vWZGH zut4X3S<&uE$Ed^GY|#0Ia&+%^A(gT51+{NBLdki#aAms(-Mc6fHq0_XVQe{+=FUM+ zmxR&V{N|w3-M?`}oiDwyU;>N`w9%-42tC==jDI_BM`LnJK*L>%{pd?XZoIK)TuUflKk7Qs-u zfi>JOlV|VqsZIBF@M~F3JUp*}Vrf6cike&TJhf>*?Abotdu0RoPN~qGLoJlBM~ZHc zFs1IXe^Hz{K@j-17L%8fsf{zflP7H}aE1Iw-0<%y$7yd8F1wkJ>yDiv&Iaq@W$d%K zbn_2R+!r=G=~NRIad=4V`T7>uOU%SKlRBv9IgU8m!2-+cCQ}w4tMG!QoY#9GI|PlZmBa(&8GQP=`Tp*|F&x*le4N+upj{2^pw93gD}Y{*P8 zgePM;(0h&tZQ7S1F|Cp4-+GJ;A9)V9O!~>8(A%K%Tm$wL+$5Szg{1z3GJ43@Ko-AE z;Yvp*Z1HlTf1FFfP4^q{m&PbiD>uiJN6%tj&P-Iztc71Eda=~#0qiL85GHkX(DMpC zN@g$t-%)&rUB>Rw4mQ+NAM7_s(otBi5kc3LhN@V4eB;ou1{sZ^@!#(eNU*~xq z&&LffBjrZBLBGbF4!>atW;sV8TKyI+lMClIk2?+_KYVGdaV{i$QR9vWej=Hj6QDR| zJm{N@hjL*q#1wNdeR?${3e@2F*&rHg_8Jn-e3Cfsd<T3u)@>W9f5n1}l$p~`ppMZQ8C0Uyv&$*9T0gL@Y=&8*N zIPFK|^nNvn(Q$!WuWX@ymK9`#o`go~0_(dAps1sqsM;h-X6&znxBdPS?|^D}GiCw| zefO1|O;Dv~K~1RTp+&9TUyElPFTibAyD;U>AaZKLDO`N;Chn^&A|i`fSiT__LpOzS zG68Om=MF!_mW&2cy|W#26t%Gag#vm=xxaSK1Fzkb(Eb?>s9<&i(NBqOShyH>PkTW2 zT-9ZN+~euiGioed%7jgRF$0~hxIyHwDeSF-ElrPjMAFPRvr$bm`7GyJ`lHp1*AI7K z=C8cDTjn;r+%_#1=xRX=eak4=P2wY-MZudzbJ_TiBv{Kv^Z$8h@CR)s@OvfOShC`H z9)EV>BvT)@OS2cMV;uQn&5dlNTQYvX7|81zs=%tda_o+o3fpTwfK@DM;Y800NPYHQ z91z@zTLMjBaoRC#n_h(J%M?IukpS0eUqrRhT|`dSm1O+v!5M465QUTV*x7cJKKj{3 zOB8%CY`7Z+T)9UJ9Dd`a*hqZdqQgv5PT_?5#V~J?5u1MK5p}a1%53JHqo;4`;+Y3q z;Z0v7mU(8-^%+37^%tSdk%7#iK6|w; zlYa5gW=hW9cq`KnXJvRopi?Gx_C`w9WHfOae$nXl_8vx_*+96Vb20aNBqoP#A;Zn( zaNOztXA7D{KmM6mBkdZRzQ0aH2aE9O{6yLlC!}iWb~x{jBA%WZL$A%r!j7i-=(*-C zMlfw@#`B5PYs#{1ZeywDnr}Gzfr$Q^s)~}%kubcTXNyOzl~ms_BdR$&nYY3mUM^Pw z=YJf_H+c&Llo?p9qeZE?})z8>HFe1itf&68|gH zfnTDsoy|5iWq+{GdP zsgtj(Uy+XBRQ$2KQktvj!$@~eTz%mby5&%kf9f;!___+O8#R#x(?O)(`~~Kpyibm2 z7ogXSZ{n?RmF^d)V1mV99NV8t-PP}4zqtp#oIQ}a+ez=8U^=*MRAFxhoT8!be{gO7 zW68aam$YZ?Y&h4DiMmf6sDXEcWdFPbRJzuHx_JfMviPOgP!NloTXvHuu?lW|nSkT; zUAQKXTCsb`6>MBkO>SmCM$aez==a>a)KlFW`{s{8f0^SneS8Kk`{jf84}8SgF>0uO z`YjnNGl)f*8qs|l|DoT@p*Tg$0)HwPfo6jA8gF!In0Jk%tn3wR_#@3rojgb+gk!Gl zB^1xel(Kbp*!Ck0qvCFHVZjcvmAF0+ zX4!jZ@wuO+bEd?ExAO^9`n-w!*X)z9aoHTU@lqsY zBpC5I#mfBYPqzG}1G|`>>v+B{^E>+6E@6>-f5VLRhWtr|AQn`Uhi^6n@F%29;%ZMB zW_nwh=}-TG4(h4gew8v3TzCts|GmQRF*XphV-MQ4p zgiZO1YeQZT%`Fe{{o!#mSpOyUeJ}@|_D#aVE9JDN?h9so48_*-nrx-{QQYY31J8r> zSnR9oG{ixTsl=b4L^udPS+9lM)sL|!{UTLat4>d?m0<4IUzo2ECNA(hg_}NK#WSah ziT$G~I3c(gJx}fw`57-1DJ9>=J@a0X{TW@j)Z7xEhA3cuOc=&Cd*H&dTv|He4j%j$ zi@B2(+25%i=(DbdxS9-O_q@WX@+S@U>&0yBv-QAZWv)<|O8MrQ?D{SK!md z*WeddF8TK+5`HZ@2d8t+Qr}6FpwBW1-deAg>m+(p?VTC&+-p zyzMZ1$y8Y1euo@5{1DoPMS!i*Kz<=V3;KI5(Pse~yl6xeSvXdnuRJ|O8s&@wPK(1; zD+s^x?P>DbSw!CyujAhp4Pg;3SBPPX8S^`6!FO2oiiMZQv&Y{C^RC%Th#H(H8Mg_$ z7vY7;Ze#fD#nGq~F+$1&sj|=tV@7PZ@n;X%vfpt(;QEq<{DGvwEW#bwfv0QuQ`xs+ z&|p6{TiUOY8`uMFrb@hv-FGk@*CICVNTb`uCD4BAH4NIXhPNvsVA+Gqu&k|}9?-S} z{gX*xQQJtDPq(HGBi@0}qnmWqh5PVoS_FB!p@nSRKOJ5^bcPwrUXu^`U%;s~6!t`@ z^Ly$JgYTQ^`2Ig#{$ICpzg zgRAia;^JQoDSmMfnIzBa8O($+@g-D&)8SWqJ4jlVs`8i1-NCQJ32sPxLW`D{OW1}{ zG$H&koxk89eIzu*9Hl)pm;Rx{=0?#knghtok4E%;{4HA0s!R9VDRO~XPw1V#$K36( zK5A%npT5rj;1E?*BI#TDSYlD8NCIXxJ1S>SqNGj>@ErS*-jn}rhVkRl0Q;4MJe}S+w0htxI^&vxxM7P z^LV28Na(OX#FXw<8-W9zqN%FkOj11YHyx>mR4r7>FE476m@KNMcAE={spyX6)-Y4* z`TI7#|6xAOkj@2$%`Q|{<`G@7D4YgcsFMhnyAp#>b+qf9BE_{+9JAx70ed zT1z=AJtn!n&=kM^r%g8QJBr4gF?9Ic3R?bs1ZHRs#BIeI*k~ku=4~5-MLK73&z#wG z*67cYiNDM7!6#i3XLT73+7vKz$~rpo{07N=_jycq`CH^}E76;~?qgTWW2ok1;r{PF zQkWTqsS8K)>p)C)oRMd9mN;R|=0Bii|CMx~eTvtgo<`&5wQ%vnT-Y4Z2D95ILFbrd zxTtbIukTWTqf*MzsWk;+ITMh)e~AgKh75gk545cH`5`3%B<$>32q;ZO`=~mx)5T*X z<>v_u9eEyK&bdO~9-c|DbSY{MT)@Q~Rzm00jp$JjM%M1wFG-eWoU8xokZbpjA+*ZV zqse9T;U+Cyb5#pv3=ObYPdaBbXrSrOJT$p{jb1LgBeqwr!vI|mQl~A%-R~7}`1GMP zu@5$;Kd652TWV)+Lb&MbPJ zWbzs-JorJKEQb`_H~Rr~I`e_1=8Qv)H#%71VTjhE6S#1$Ia;)&;Jfe$8az5$@+hnl z=hYwPe4O*}fR+g!@=2jTxVhq$qo066>;tZ->an;g_W~@v(heR^ZRoV&M_|{jEHH4% zq(x2Rr2UvgaCU2wNO~VTzB^k1K3zBI!Solf^XFd@r1^&^UfT+*#@fQnAPLd;tc4}t zBH;E4Io@B-1@^&5y3JIJ|F`V~nKwX*U;Aeu6pb7Q65UDY(LIIt`=IPl(w0H5;11p| zd1s)ENR{;0W)b;=Uw+*AV2Jr$Ss(}9{C63RXrD8FF6kV9Y(Vc zdo|el4ja~VYa@R_IDr|Nc7UR_FJG_o7e5`EB;7e}^3?DsQ;6?PcWz9CSPG| z{axCbd>^(-+=%?X4l+pEaO z$!ff1+j{cGRu;kr1>!s9C$K3XjF8t~x!aeofL+&52)mgsC<;9b`L;P=)K)+PrwAbL z-woI`#!FIE@WIhnt^#(X3h9jWexAxQJ$A|Hc8}o>2t~x*Yyd8YnH5=wWcfubZj^f(;Q+)pu}>&sv`C z2!$ z>kTq?@kGLB7vsxiH;Ap%Wn8WMOp+fLPFc!BI_k0v<{b#6BW~y6^*aQ6W4iF!PaB-z z=>!h56@X$eWkFummJ?QfbDuLOS3(@@T!A0E*Ucp`}P~aVJ{1uTr`TSULQb* zAAUnuMYy8x@(l6FVR|@U%2b9N8bJ~a_R?CltMv8Dosz_{qa8mt&qvi7#?^1sL(TO< z8a>OH+Amv7H)KwyW1O^TZoWP`x$LC2c44^hs+b;IWkXDb(Ksi#h^{)k4YL>TmFzqI zkRH=IK|&e`bKcNL_Ep7zRp3H4+-Ve_J)lf-cGzrwoBv{_azG2Y3r@V;^bh#O;upD+ zGn6Gp&){dO83U=B%Vd5@GuBhrAS9;?BI@mV=Sdydc1VT){<9J59xY&&M&01HRgtf* zUde{Js4U&2(Y7BU{@$fn~@0;Ip7%#QW1r$&_LD>7a{%<>Ck2ELU@!veN?Z z?A9UoevY7~+!7iW9q%Zk*W;KrH5yy=Wk~XoZu+3&1YLJ?3LV-zmTtS;A?fPxm&pEf zLDk$q4rlnsh$eO%qnC|SvEk3DFFWtzc zAJ#czj76biQ>Yov+NX@Gdxw(tyoHq8xq!x&Ult@P43s#3-H9EW!ntWXl`(txPP$ur z7Pa1Yn1*(=OE@i43M&DX4USWZN;EFe|3y8YOe1x>qtX6!Hx(CdK$pEA9gPkr(+8I~ zk$*q#VgL4368%}Jjt$z$_)q&8R*o1=MlK1({#z;7zWWGSRIGA-IBlq-Ak{$OqF|YH|c{te|BZ5cr4MKZXk#dZz z_+!Oy{w>dD(zuCw&y04KiZ$#S?=@E*(F90a>Ndc1Yu5dNgL6+dFgM&@;z@E_%# z@%x=P(rg{RvsGmS&K|_eVtd%UUYmI>e?V(Z2eFB>yr|*YCOXo?6{hz+ z#PQQTXvJK0Ny58yoTu7}-yEYHegB@sbDLA}=#L8|%ghv`P9$OHp(rj^YD`%6z7zwl zCJ~LvEy%^GN*R>Dbb)aQCQUQL8Yc-YZ?48N|Km9C!9Zp>WD#zBRZPB|8^Wwr8NF?$ z$Ub^#;qP6BIO~rCl%3cQW#?3=ySo&WJEl;0-$-he zroi*>!IBM+9*AqNNZD{MmNu%SKyb%7vMc*2v2+~(pE{qAmTmLNlM7+6w|*q#mX(9w zngK9uRw`9jdkqyO!Nh8OHF$O{BeGfYFtKqUcHLgUw}%pH!d;_p3qtq{ug0=J%>+i8 z*|K%lC-aFD!pXdy6Io}fEH5+SFq!D34lwc# zaqAD$k$bj5R8S7Iovo#tt6kuo;tp`}il$P`lU)B}1CpsHXuaz%$+LsS@MY%>I;-I< zw8wnlYI_cluOq9-Wyw8a@4u9ch))NxrV$(tdJT#z1Yq8%hBpF!f_93K*e-hs{ViFJ zhbo)Ms%hpZ=NSp!Qa@wW%(EP)x*I%47DM{-^%4_Rx!EwPYt%W$Y4r$BRDjF(Z%Y;UlvdICF>RnYNDY1xIp~t|RZlHvf@XC&xjvu~ zR;V0>V)a4%C6BrApRkN(*XZ#3w;UnH2bK8|C$(X2J_lAy0Ba-W^Ug5^+?lN84_j~wvpekVd1el+i2D+Pn&|k`YyZX@AI9G0sHQOaL5BH-~1EAbDiMZ4{hi>qf+@kR2eA44E!cx?2TS>)+pEMWhu{cU~Rqs-9fHV#mdIHS# zz1ZhFQg6!a(R`+pGQYINk>4B|!kSi^@IltSxUkrV=}0?(d+T+1_wDPM(;g9?E?C1~ zQrr&LlN8vRaq3L#(mO11@Z&Ovl#o9~LR|H`39nxt4kvsM;$FLpIO6tqV*YI;mY<2m zRsY_TH$4-HP0f4UrTdJu9H~O}W=p#NXAPBEV2!(qY;gaR0_wS-6(eSDMbjH?T15LDuYhyV*Gv_qj zH2=ILHYSbEO&yI(S0Bf|^LC`vHxUeVZh*{030P(`OuHBj>I-J_)^Gn{MZXM|?M~wC z-i+Wk+$zHJInOZhoE2B9KO1Ull(5@;HP>Hv8!vejz~GLlkfzMx#^5-3Ca%H(R+YH= z(hc%RzX@VSuEWA9e<3b02d)SQ@;8G2QI$1?jxxv0giLP)8ecs}k1p}%{GKY45~W-k zw_YTPxpBz9@@5mP^R8jVu5jS~kPX`?%aH_^2p{2jG~5uBf$ z25$3yL-iLFiWMGqNPG@B(dUjRwopAze@RwQbMCr>Ncn&yc8d`m>o<*?T~J14wzNuO zD++0QQ4YnQr)kq#Jq(yW2#?+|rdCFq=>_9#aj4)BRfz1MYB$U|PwDxzU~L#pe_18m z@;#Hjj4{JY+9SE3rr+F%WGAc>&yt!-bctoK2W>Ug#60!Cj#|~{Xl?O8^l((*ivD(r z+jeN+q^kL3Xwh_>{0*ZX@9*pI~qP zH@MZE4$F=Qv&rVMIN1;FSCCuz>#$k_pV9Fvn zW-Z%;U7h3MSB)IR25GZ3%k<%dlNRr~vKhGNZdmBGglauojMdXqIEBU|#OH)BPS0E+ z&6Sgh{h%k3Pwl2yoimC{{ZmRS&;V!51hUZknk1-i0;+56CQ~!L(Fpz%RhXQld&xWc zrk&E+_xh>rupoSNznB`8hN5@73vG2UqT?i47%~qC#vj2k^2=y$NSCB9)0~>FF+f{! zf+YEyn4A4&9u6`(j$!t@#gE>fqpn|HP_2uzxUxYKYFs=D7aaWLsMRkPhkWUxhgSuW zZ8DMgW%^25FQ&9KT?SirQ5s(>kGai&Gj|Wh^)8gvH;z+7p74CR(7SS*Q5}j5+T3$2MUowJ=;3Dl} z`tXK<<+$Z5Lyc93V3Jx1N&nG+S|#tvi)*)G0!pvxcb0(E-w@n+8mnbKi%sT5le#Kt zcXfLbuI5gWHa{!6c7Ypqs_Ah?e`V2Z*($7gvWay2BuKiY`>T?bCS?5OC^R8Msf@~P zT4Sz^q2q?2y^OSv_bm~py;H`M){EO6%?}(xJ~RVotEx(#?d+hPn;fvnbsAS=r+|91y)j3u#jUp7BpKSF zkGU5ui0O?u{ITaT6@71~5e*};FHHx7{fschQ`+N69gYkBoy3VrgrrG23y^35*D@g^G41m`IvX!HWk zdG~HIdhtEtuQ!I%D{SJboi!+Kj+Zo+)sh(j!-%U_GRd=xCSO=8nfXf#Sl?SR+D3*9 z=#Am-qz2QaqFfS3^{OMD=KScnU>#FhBK|Ma+uMuu_{h*Vck6=xU!a?5DkZm-4kI9`Q7`Wu5{n$soD}UmQYOgyO4c|nERsSGH?_{7)zL$DA zNY~kVIru3TDe+QNgF;qC+RjaJ7#jRa@^R!?h;D8mQCj9))A&7-#sR8i^tMD!^x`DJ zQE_C~T;B23kZ4ZPuTOqnwRk3?~=BD5u%G8-MmOvth6-^9lKl5GD4A5 z?%#l+mlE*pz%xXx+7wgG;;>>*B&V+yCpJ(QNi+N`Qp#%a^0gLfT+mIG-2HG@nK{Nh zs-fri6ym-!JEUyHcl?{^TCeT{WLA zY>9y^Z4>@khzfry%8~E*9l{=-9n1S28NgNtFJ(Dl1NrnP0$x>jJ@bE6f?&IvxBhAc z(KWJc?+_JsWLh&?7yNNF9hyTjALgO4UNgSRQUP5Je++$+fvsFMsb8aun@68QRj+5H z?#V08tgr=r@+cYnxC$F(Or*0UjJPnrx<;q*a@H!j6f@1{hFb<<_M%DPd;?TkQs>o&|GXK{6) zG-u7Wz_1rrP-EXvu4>;}iOr3hxMzxx2rqudyUzyWoZdk=xhepwJvr>&)lBtg7h<+f zEJg+Y!n9}h*!Vey#0RP|ugjxpla(U-&@mA&bz9(`7(>t(JO#@ct=w9>B*zU;vmm?j zBkVk(OiP0!;oZX=_>`PY&%0WKS7s7q9XluKU3*JhsCy40qlI)sX%pPBQ-sOhUBqtm zcG$ek4m`)+AiADUz){)(S}yGzOjth$zWLpzt#(?xmUcQx`>n_?4_A`zTdiQet8_-& zKcC;>RLo^{R?-C*H}e-ZyD-@Pp6os|nptmgo%%KPJ)j2qIcQ8br+zV!IP`FSoZT{#tg#S{F`tqGbAE09m$ z%q6qe--Pwj&P}!N40_^m3@}aUdFu)3LeHHEl7Sf@617Oar5|z7Ui%ny`zmSF%T^ft zs+W9B8w?ua!*J2f9o%0%Ac>wYL1{t^Ow>~1?|A#ccGIWSRLbCd?>|fIz9{qjM+JDi z-vMyD2Zodjz>gCRU~>Mtu*BAwSwFSq_3}H{vA;8ZK+~Kh{4aR}wqM(c zZ66!Ne;Qg0H@~iA%N`%4$}NB3(?b=$pULotuGLCDNgh!hrF#GsA0XkeHO9sq0sT8- z@E6OW_h1gZR;9uDt+FVSIF)Al_rRf7b<{vq2YK(Ok=dqS$+M!xFyoU42TdexQ zA|?`E2k7vxrCnUxm%i92HsDt!+#@)72;c5YKYr_V+3 zgACvc{~B;>E~UWUG5K&X_yOJGI0uBAN?~%pfh1jc)xle?3doo``s7C+Tpl+81Y5P? zV2QMYIdKJuk5-egA5S5qCKa@pG9Ns}4^Ho|r@_-k@XmY^DM}j3U#jwefPOF@J*fRcy~%U=Ru^it%Af&y^e)Ho?`_w zh1-4`(QHdLK0Q`Vs+D!H=f6nY-}aU)o25ygb;Wobj zY9H`j{2*JpN67kuySg7ZTK|l~w<>8EG&hwL+!~7!vgc9HZ6jCaKS!+UT8f(Yu9KXN z((GcK3NH9Ai<6Bu;K7$RICbvnSR^N`rfx~23h{PSU{mzlGx|%HF?FpJcQi;u5 zGYa+InBuO}cCbZB12QzWaeMo>b0uRBkq7Jd!K9i!5`o+>NIW+H)U2%Ok@jIk=qw}+ zZ>~z7t7%Kb>Wko{qBhkXyBIF~u_0@7EhIXM-lV&}otwHYmHxErCrev9$;8D`09V>b zVpAX8_A3RB%(z61?w)~(8RNKdV;T9VriHyeD*QF)d34QjMrVGR%ZrqFrhni$;Wmw8 zW4lK1zs>f*x;P7#H!}++RQ|^$*PenQQkPZa^KvYzpUrFNI^oLk3M^&uBm6l}>XY|# z<=wB{!Bcy_!{H8&-#_mM)}?>M#s9hRE0;cjbtC36OZn@LmtV;9ul)7+Eg=q|zr0kE z5TH)IeUgb_@_wlA-$Sz+>>&k|K&2~zR*lPYTrqnt>Df4ox?h=1FGL)M#BYYwLemox z^`~-apOYktT^d|$*Gg{A4=L*;9tACluL#?G9%T1vz<{}c2c>7b+BQE)?c;QC{b5fH zR$eB`(mmCRx*tTlT8-R&<3(K8q>yW$!=>GlbaCi;MJUhIgVgEHbgGeobY>g^McPNj z5ix3#sEM;->l^_UI{Crw6DP>wsC0>$Wi_WK8AsUtLV7!D*cD*&4{-OzDNg(fDRfsigC zct(`a_bSrgJC_6caj}v_hk4=(sT=e|Mjd5G-owieIv_Atg!|i%z`vJs;6NrNUa4)+ z*>(~xamswx$(8Wi>j90tpvP+-$RdF1{QmVia3yFajIf%AeJPXq*BMW_qk4B~vE^p| zPS!Lwr06N>H5tP^^Bnm8{4UA&02?;3c!-n-noI^?C?_u(C$Yl+_F%B3D_`iFgn?g7 znQFWy3x03GoO-wL&JNa0X61h{t#JwO_rpjixzWX@n?I{JX=Y!Dtg###bWJ0^-7d1ZYL2E|+ z1|Q$|v@oLq%HN$N&SSgCjL>CpYl#Q6U#TKnRt?|}eU1X_O}hM+YYA{UdZ2zOM`l56noQvc`L<7iA&43!I z%XE_eD#$ApdH=E_YZ*s&kx=a!QFTZcgFBs<)Ei03Vp zQ@ErbYv?D(VEz~1j+Isip>3cs+xo(a|F6qP>Y=n|It!KfnMvj3{j6v*aGDeQxB3Xm zmCxW4LKa|0%rJIri7ay=c5G$(Zr(3sJd2MKLd7v3zPM{JJAU7b-T4y8n-oaDvB3GW zp2$4OxRt$-zEX_`mDiA#c~auqRY}+0%!jrIHSqinpjPBE*t<6qo(o=5%`|Ng6~w^n zs*lvH)sULSyn`NTW_{)?!dmym+%R&VcuW8|k~$h@tCo_*1Dn8=ErX9?%Dm@;2oOlU zDGoa|c-Lna$>}xne9-#Iq}Qo~oZIP*!2?SnIX|2P`K{zM-~%@R_NV2#T}&Nbwc+fwCsFIXGTZ%e1@7MckPL0qWAQtdbl*HXKrKt5cVk7miDhN zqYk>0_~r)(fsS%v^WP?evEpdnBTa+9v)qC|e?EleMo-{Bu`XQDyM(Rs`vu)I9r?Y} zHnOx9sh?zbAfKZy5BXB>L}HIJQ}vKK9845Qm}Vi-72Ze3rJu1j&kTNaoW%CSrMTgi zA|xy)*q?q8wc37?X@;KU=Gs0q&-qLQmi73}B7w?ichk#Pm!a#gxmd>Er(aeNVBTww z;P?Bw?8}#UR5#;`acvy>_n{BbK#E{qU%Y$ivgS0T0sAX>9FZF3o&_=KaOcw1W%<% z;i_5-NzkKx+|r@(_)A!i1)JuQSjR1x)0BwU@1Gza#*W0cj3YQ_@gMHKSYFcq4l%~5 zf_TSQqq+49y2R-vebnoQuXz*9SHDby;t}cHAY8HU2fnW~N6nI#q;s$eTLl5MX`T%G zV^~X<+K<7sMdt7^&x<`7`i|@Ei6D~r?MzNmcLVh_;_>twMZpq z+i;ssF+&=gIfZAh(qK%|0@isW2KvTM;H`|*c}bZi@20SYeHmcJv+lq6QF}Sd>gfl$ zPaJQNw~0+?OUD03Zs6m?&Xcz#gILY{A#BFvR_t>Ia=QEpIpKL3k34&d(>CgZ&?6Xk z&bf%$Ru9OwfvWh@b3a}Sct<2h2b1aTZ!w^mjY`lFu-5IQ>AX; zaiG^z3{iSOx(Rh$ydV?)^)~=%te03Ttby23QvcbGL$ut*2-I>DfV3=;%nF>&eXY-j z*u|G<$+^35?O+$NeEve(GhYh^HKy>rJcBeS-GYaAw?LlpPteaE3kt?pX};8tZZ&oS zIVAOMyNBw)(eK)@{>fHk<3@fTKb_TIj;14yn6lq7=6wEvRg#RI z<5~DG9X|Z)Kd#E~IMJM9&v4XHe5d5hYg~-S)<8qHXd$fLyNcB+q~rZfzu@kESzbr(4ftKUAU>6xLceU4I{x<6 zLq_uu3>o7GM>Er4P(Uqx{$e;(8l8m|(<*3HpcVZj-v9+Ih4lTSQmBYnO49b+Ci^Y{ zc$!#2k6AJ4pYaNuA_8IA0VQ5OZY>C6r=Vz<7N0I%972uWVU;PBq0B} z#P{SM@s0y)IKdTtOx)4JEjVz4=6*d-U!U*e3Qg6xIRQrE`5T@RiSkv++9`%KPt6if z@%dCMzKnGE_0p29?KE$WAx`|*%Ds?(M0W{gz^kPkt;`%?@lH3WnK6NR8_V-4m%?ek zy*0nodjuPyas^{%OyG-d&qp8oelp&MV^$l6^8=<|fXeS9SohK#NVi@CrehDl4+A}3 zv9S+jPb`BeKW^Y3MH6;c_bZ%IegQKV4`GY=flMdJiVqJvgb$=nj&JWqv&qee=*?bL zV*N&+hM*g1?)*r7;^igh>Sp3DuWULhZW&2WE9J5SuX09rPl-;qI}x?YNf_Psk?V8x zrKjFT(SbvhNXO4;U-n$`)x{ZuP1~rf`e1nZrkg5Oo8aOGBOE(0 zja0D5RCcZnlpUH)r%j1*+&lIhRmxjPbnWg^`x$-WL}NwlNUosPS4_!j-cEeU#EgEm z?HBjk6RuKvZ_fGFa@Mkwsea)p`d^+UImg<$)27;7f7W->c6OfQ;$&VDQ)PswgBjg4 zw~5R!zCeYpowUE=)dLY!lVYUs0&+`dXy+)<$^3uU@qM#8Sb> zC`0sJGKib=`vufIPGk1&Ss>cS6VC7hRIeDtZ0x4s!jO1=aJLGZsH+2--TP6iHHiNU z+I-sYbTsvEgteaMAb;?8#~=TW^W#@UvrYEhL}Q%PT^K)}m1_)xSARDEm3|w+Xs|cF zEivX#OO?83O6?d8wPe;BSA~POJrm6NbGOjRfr!&pzZB~J{vaqnK38;a%v}*1d|ohc zXpG3*AYat6Jh31%e5oL^ZueY7E=#50C2<8O~@HnS#iWL_>6?OSxjVe-p3L7&}QQM=^2pi{bNn(;ip`W=w*zW!^7RbMB;Pn3fqE;?bWp2i6X=f;tS;)gdZM^6WKm77WtT6vI{D{ zYCoUT5uN#xYOfNqOKAG?vS_x>G(miclK5EO08z>M2+?N4NReORZTqk(OGTCqMq-`C zk)m&V2Z$rxrVIOQ2MHEOo)<6g4z~|@YbW+yx{2tLwqJXEQC2Y9+mS|NQ0lXvWfYW=BrR>J}pY(%F#bVa)wGLapY~uAdF^I9v;QmVV z@W#Yh;_c!Og$uWg7R-7(Se!F9M4YdXDolz!DvI)&B&xWtZa?nbCP9(qZjnGMSMdA( z)55a$S0YbaxxyGVUGZTLUy)Oap=j_7J5hk-nthn`r@6a=TnfT>QGtrJWg9WP^#);Ji=ZIBCo)$*^ zSSxh=8!OT=yIMGDrLjZP!olJQ#ZJNIx&flz+6a-6Zurga#hb-5hkJ@_txUH|L&cW1X*9%O+Bkt~}pv zYt?yiyTUCYJ4wWQ`^O9Bm}Ln99xD{;TH3OAKYOWZFGHL4lbF+%1H9Jd5tuma0hku* z;97-qMCRj2-pXMiE?E8uo&G+ALv}~_zb0=avyVBk`EeGg@oyxb^Y9=)CoBsjb^ox( zB9tFZ3 z&T*pkA%@}`p~oB|M!1V+KDHDIOB(DCPHYycWLXKmCi4!m>w<+xnmomEw?I%6qb!zn z&vFRtR~2a|WQxr8Cfn<~jS_i>I*AQa#Uh*4A>#0&i=s<)sRHk=^J2SYABD%FT*RH< z1P-DcC5L+kE9}2@JrrHt<0e#WOmIlJ^j*~AJXc(_ae=VnZHXxHoRY)e2}*@33Ib8O zijJWEpG2^D>nVGg&|f0=zpDf~3C7}0QB?53_nOe@QL)JH-U9nFmLaT&u@g_+H$dEG z1L8;2L}Z=!)GlE5Iq}rLY>mZ?G!CSzYgp14a3kB)Y&y~mSRuolw z-xO+6UqMyD4^&!LL&y8hgr-F&n9Y-U{I)Z(B;98NnBiNzd2%xZ8;|B?3i?o``X=$H zTMMp3`T!#LKyt|>W*#Pj1Ql&Qs78r*?j6e?9kH0z?$hCSJwGeu5j>b%O(Ypbm0&?j zJPXS@RPfK<`CnIPm#9QPhm3f;hQ+LGK7hanmei@t`yc}{(?J8+(rG+CfvJ7UMLUog_R4mL{WEbL=*o< z+?g=c^mSppd6uL}k|dSJd(S@S-g9rMB!of~X;Mj&kjNOCXUb3-G$>N0x@Ye*+$JHZ z%poEDMdr*@ro4TN_e1Qxp0(C*Jr7?QsoZXc#eb{lO+qW8hnR(W&a7sRA9p0GqSBaG zFIs8y)bULJetVn~E)r?Yv5@|INr)d^`CLD&U@fk&l@+VqQx`|(3hGblYHnCw-S$Vt2vY1*Tgd}^wp=2cah4RKS#3f z`lJp!5~UFbKS|9tuC1SF3CS}#hs9g8=84B?r_`5Dyeti_>A(#duYhnj0X^Uq&0Mpd zP5-n|ppR|4LU%8EO}9&J@lR1fbojyhbmhD$boI?7e3JZSI=7*WIDY9V{dcs77JSN} zmz;Wv)~W2H2P~>-v!!#8%FAPTD{7)oFfZ`oo2fuNOT;KiHY2S~a~S6{KJ>2y8AkbQ z4lZ~112KBx6e`cx#n-PYW6qSGqP1j~(NQz6Bb`0|sONAi-Z(LdPI8&S$UHTpp9No) z+)mUa)P8gk`c{EVt??n+KPZ7VRQrn@n``L8ZUcsOGbB=<=+H~McR-h-%Y?nNC;h{H zEmNQPoUnMYn^A-x>Bkq_=it@1P4GS<~9H zbLqq{6$oJt%$sxLh=vC%P|u-wrcJL83}weM@$tp9hv5cVd^sFsCr_>4yYwcPlDA&G zdDF#uLEj?r0`OU6{kBuY4GQWP?4v~6S2~E*$9gE|%!i`Kmlkn*i{Fbr_BV>uw8zzV zY}_NZ6!nUdMsAY_->wwf4hPjQIXXi;Y}CV*`vlY*F8jicv;8dn;eKQ+uT@Su`}a=q zYjrd69;rfoo3jd6p0bF3Ha{I0xRlYVx67HqyeG7;yf&>f_XvG(MFU+QxL$H(SeY?= zsKNYx`VgtdDTDKWI%v5_S0bdKfiCitV;XsWblCP(*q9>Ad?Nm%JCma5J;qA()LTNP z`OY&uwe$%dwl$e)!SC>Y>8xeuu3tiHAIP9h&U`~%dVgs9&ItNJN;uO>U13W7q|7zv z8O*AXBXnx)OZrNTCVhGT5&Fg)2PXexI~{6$4V_|AHSK<5**iJLw%G3qVRSn5tEhJR-gm+*)`>{DjVj53Bf8%FnyzM;RW zHX?Pbb5UDDW=6H98UP5AkZ;uAzVcF7{ycnz_QP342(U?ym@d1n71B6hYcUo z&qLG%{>xs`@6!C}$?FQ~&@D>|>ymYhY~%q(23_YZ9r{45u~ueItqLLLG|j`c3-g#T zIXOY}+W=-@?P^i{pv z;B8U<@B{JJcMs}Y`oEA?Yx!bQbU^(5#aI?rkw^U{Of2><&lktNmPlu*jdzGwU(g_P zph;Bz?2Bmpu1)oy;W+NJMS{5Z7h50rs6>P|RfxU+d5I^8nR=5I$GQ0FRwze)1Cjht zo~SnYkEwXKjOfS?7mN#?B$&CQgZaC;2VZt{nBINh5;O2P3*T^UJ{=UfhPgpa5v2BX z)6uLk?$@HlT)JaSyieZE*c;R{OJ@wC3m=RG{)0DZ>pl-chRdUVO|NAvbqPV8UnDbr z*BbiKiY<(dbuRID*Dry5&{v{muP@4=k(BZQ>%fM75f2#T`mn1aPd_?@fs8JGP< zO!CTBRR2hwe%I*8Oromr>whXl3ffPp zLNz1gfYfMnTO}g%S|{3DF_AGS_=FZ8w`1bWUVts0%5>%0Z2Fy{8;*Y~LDa1%M#}$6 zgO%%y##_1 zZ6YXRA#?5bUHaJxZ>InJGe)7EWlVKy=r}nK;+0o1{m}Lvtv+E5eQsSpvruy>GKinS z)OAdRP<>3qYOG`OKdfTN3zM1hz|TOTP=-N+kLk}AeoSk$3ge(`%g9`vM7$zq(#twz z*v6g@c;4GAW@twh^SGQvttn4v;jRPqOP;Q(f08?vuejW$VNr>ygZYuQV%Mg0>E0gg zdhJVjVzYsHVzujf4bFXzqM5wg;`Y$((#7tM;@!Dt>WvZ^@wu>Wu`p!5bkjYPYFRZ}J=WzMBttCX141xty;ldB26l?v=jd^h8=QVNV+3z^=1C+O~NiwW(J z@yyHU4BB4*DKG_9U|n+sbEkU+SS|0tH0o}k=gie%mYs60uhOHFY`5|YN4EtSv_ zZ(5j)@lTN;axtL=ELq=KS?Eze%g89N!tW>QGc5~-v8B_}>b3G&miaD~rao|Ki0=F( zKB8|YM!Jd)bME#?z27H}`O1D5Gk(9M)88xCUk*2_H=yhsJe6|Abv2#!7b=a!-@acH zOQpZX)6W)(ZyeETxU)drVeskr`UK;D^&g2TV*IJDc;;iLhJiVO^>3g56kFAN75#j- zrGE3qH}#V;Dw)UgPZNvFYy|k;en!qCkEz-!By4V0GVe;JGLe_viTlB$^egHOD)^F0 z$kfOaCYqDjz@q0wS8p;yKC6P3r)ue?cV{x?e|%~68_tZfo(#*`J)!G)6IsoR!}R_; z=WwoOCA)Y<8=adZ%N||3j_AA!X#@5r==D}%P2SBD*!_AW$g{9#eSe>3!i+PCnocto zx1C1Ir3Da|e(N%wHOT^_Jr9YU@^gvYcS)@K;%|iaxMHSt=qI6d%8PM3{D=9ZSW8rj zW0@HzW7(zlDa?!wo7jjB4+bAG#Z0*jbBZD1!h6rVCDElfZ1|9|cr&*xOy-phB;)BkS+nIx0jBS~;@=c~x>zx`{YtfItz4j4(kIbis+cF7! z&UN^GVKJ>2&_K7;E~zWg$7uOQ95mf@5FK4U4*%1cL+7@gz+YaoA~p$j($nr;0l8-n z;d?d;X}&`g9TSoQznP>HN?nb}iShy!t9TUoUo(|!q(%o%PDUQb;_;f(939=%0nhnu zp%={7CJ2vSl%4EFOxC|o80(JWg%&KT{569~6of*P_U|yU#E2M(sYUblJAtcz%V>}1 zhRBU5B`*9krpe#NgsxL6K4g=QWYqZ3Zo?U*ces$)-Iq$+x|E{mBkI@+=O1`o>T>!U zY(}qyE=14M44O1ihIr#0I9Q-hDXl$?Vlox!QxZk|QQ$hsh3V1Yi&qc4`}1j?_wouf zQ|k-LdGyTQ_GSz(ne9f#L?XEKBp2VPJ_#+p$CLDGHiLxy>F6Du0)zIFbsJ3rd9fKM z&>I&=7!?=ER}-%Xdk$vR9gW2Wa_exK7<~E&sBfDXoy!1@B*0<)0lM zkMA;2lF04+hd;nEyn9d!&AvN8E$y$Sua2-VB}$i`7Uo4y`uZE-NxgXN!e(M$9FD(z zT~&K?`4?0^?*N@?GYrn3--`CUUqj!RC5ImkmJ-?@>+ofX_F|_$0+`RoHHofvpT_3DQd+d6s=(fk`NX&k)>UxPiFXB52DsiDU2y;tQ6|HC?VWK(7XtHF#LhIGt@ztmRW9yIAoJv`F<8;Z8q00-kH_^SCM zWgoQ(8Ld1(`E4>HkjE2bJ)1)X-EsIyP=>Be`~$QNI`FyEf}z!#1IWeX2097P;%AT_ zESQ%^m&pX+L0U!hIrZSWsWb1wt{Hw1Ez~A$KD=U$w+skthGXjcWtv3uz#5T^%XU%N zx>j!O2w(K-&1(^NAYS^l#z+LtDN3VGJtRGR<;AHkiuGLwazqK@PolNnH6qR3L!#{p zPWAWR>5Dc$0@DAKj!7rP&yw!Gd06^S!>0b-J7tlx%5?F<1(~7&(p%b67%JT|wFEco zx&X6p6Oeyz1HGbYFJ1Jp6TJ!3qd|!-+S5`An+%fB*K8xY^yY3fuv3IJU5S7WZk#TdkTvzgRsJf3wM+4%?|tgfO>=$EQWKc2gss^_L(r1NDdxnksai(-Qj0d~IyX z%~qhP7KdM6=L9a#)22-vRuc2q44|w*Cqn7Z0KT(Wh4!h}z}m_LgfFj+-e!6i9e`(O zadVmY@0;rfcrFcViY&$zrt0jC*dB$kAOqE4j!JE4ZTgeshCx-aFWo( zzePNTn}6IwN3MMU7e!7Gk#{l5FrAjNA-odj2FoW@ zQ;xP`T;6au9zC=TpE;9Upgb*TQpuY`}aF(i}e+0K)kwmWZDa%gD8-+imjCz`5KDH z%{CEfoi7r-b#dY1{!5ekwr5IpWq*h=MT|6TxKwH%)FR4?94|!!0@2GmpF}ndP=q34 zrMG1AxYva*#(K?H(zh}O^&mowyI!is1s_aM_T2km-TN3K^Po17sj?Y;yt9=4`Pv?j z{wg;XmGuM*ev}8f85sWFD27IFX5xNpEC{!^6Yz8S`lzPr9rTF20&k=j;@I$8+<6C& znB0|%PCFJN!3>PhnYNZV|LYI>NF?D?d)p$EU>8H;+5xL5iCQi&%Ah*PEPHDrOeg z4A<)2pu{^LVpy*RWoW;gH~v61rkwYLTbUUq+^Z8z_J2+iq_Gm=o!isN-$@!IKWiNK zTJ9M6booi{TiGQ+r^;65VTC*naz}*ci`}^Wtp=nKbM^U#GxE6fKdA*D|p!141<7_{~>HOJ;7#@4SuZ+;9>kHW{^sJ{l_@ArlK(iN$e zZNGp_;0)yM?FwS2O+m}%vzVIaSTgta3~=p-4Nz(nAh-8BfWL1O%o{L*fiYIp9oK3& z{nbg><$4%e%4fpQJFj9T`sb+EE&Hj*o5j%Pkq&Y5Yackpd_w;UlkoaTIU=+$8Gl__ z0}^->@nlN@da(B)zDqj=56@J^yVcLbQ;A>D>oT0UnqLb~+PI=?y1Ce&;1BS_WCNt} zA{<#fBJeu~0dU8}e0;q_~xxRps{TdTb^`9PWUifp`e zwL!RuNjomO7kx(%>3WNcP&XGHGo2-MdnhM$)ci%}Z|E1*pFJksTyvPrYn~^{k-bU& zY?T*%Ywsl2cTd9|PA)~#u5cn-?=tY&rb3QTMM~_;Z7WxYA@N+%G)=?=dVOJe6(}s>gDP?9gaDdeDy8 z)OUp_yIc*`m!Brs(gs3Fp$SiZxF5TVhK3=wD!eEqhKjzK`Ur<*cOB zI_I#pWAfpA;{a#;c82hC)_SsI(N)ex)rb7hae+ILKZRB?U5#7B>!9|)c>3v|RQ$o# zd*D>ueQ<52E4s5Pl^Qm#LK|;PM*sB{Lvt05@9`g>j@3GdA6xz%*SI1;-edau)1VSs zdwT*()Y%4S%j?m>inSEtjR?{&SZs2OlXB6kEK*T*6Sk`2DSu8#-lg6`SKYVJuIMRH?Ttc<#+LCL zdpkgvLMdS%l!piVJi&<2aQvZ?4cbzqO046ap~5@|+NHaNK0TO&uB=}Ie@OXA4;;o% zuH+F3hI&M-RS0o%#0VtlJz(kEQ<&M7$xQN1f(s3?;uhb#KolMM!aP`y*q()#aGiIS ztaI>d_LB2DI$5oN*>vFsD_>>lBt$#kGmH;^JxitaiTouzic(@ zUSiG-zt&?uzw*V`S`=_qM-~b4)mLySb&0Iq)G6%NC1!%*x2BB_{&o$YZsa?}R8}_X z%#U|8PVE%`7(L=ZTPZb`S|mGIC%QO#>RL6LR2Gp$rPN`Hp|#_tv`r3+rHdLJbssnk zPqcHavr=+w6}veu*4Wl4uyb`>9a`29IXSBF+MXE>o2F$rI9s1-Y>i%0ui~BPc<654o?$2e5b)CS~Kjv zq*a1W(G+H>tUPDX{y|{dEYGzvdl;RsW4yzUbn)Pl|=lr#C_D__Md(m`? zF*RByun`n-M{_L%3;Gvx0Xr1gW2gI=kDx{nj#AjagIAba56oCoIgCrTBDqKFG?)_s zO^i#E`asAsJ_cJAR$SoHWYRQa8N1!dP!jlr%+rr80+ zNwg4WbDfWkzW@Vbs^Ha28E|jA8IrU2LaN!$aDwtWlpSF}fmzoiG67xa-4YY_P5PqC{`Y!JCn0T zDxAO~i(U5jhY)MYAe}rWkh9cnxazL%%0* zCoX9Szde{mepu7W@%Cg1j=tSbmZr}Y$}CQ1W9L!qqJhQ8qiG7TQfUO^txlm+0qV$K zwF_wZxk;LL&O`^DXJH`~hhYt+hT6)EL1Sn?71Wi4>WdRW{Ayja*KH6B_w0ghC) z$zEv2PW;^qZrYqgTQ6E57x_3~Y|})&SgwVVSH9Jm+F!76wW=B5_Ig+GO;^w>7^elQ+R*pPx0Hb0Wc9{(v3{2oDLm(<|l z&{3phFAw!&c<@9AANwl*5JvhHWA&B-P(JqG<=y1qoFNiKc=%DbHa0?o^sD^gEORK) z$c1s^+_C(LTd3#vRME2iD&YRlTd+X(GXKlNE%1!>c6cvP2;;B!!8g(RNQTgcz1fy1 zF@=@Lyojv}y?6|D>knZ0=K*@JWkB6{-%VY;u7U07QQ-c_=LpW64`g+9Gr3>hKZFx< z64*gYfkXxtPminU{{Umawl zC+Q%sSu!9f@*wz{7>bN;N@1Q_A!t$Tsr#k)0rDc%FgWB0C+@6+hZ>(@kvVzP_kIxy z=fog7@)oQcV6o+Aa-mgYIMgZff;W{F(2iYh$k$p8jy8CsUS=O<^*w< zJ`c3C$BaL1b1k*b(g8bc_!iF4NW>nsT7j0)H7IWRAWXb!1Dri`sCRAGVY2@TzDe|$ z^Qqsj}o*|cH zsHMADlI~|HN$o#}6chep^0Ge2K><-`PXD30_o`v0ds(n;i4WD7Jr2IR0q~D6FN0rF zEm%-ijm6D93*!HDNu1YM0`rbsU^dy9QgEAvnWLZJ@0sgnMMsZo~U+Ky(BJ^s^Z zi=(qR>kMtyee-f*jZC5N#?F7Bcj+a{RX-9wJg^+^+R_1LecA)f&Mp|!`;Woh?~^I# z00)p!M1h#)?Nn8tAGXkQCO+(83mLZu!0Mp~<-K7iP>2x$sbo1=vEdUK+-QZr%asMZ zU9NcJ^4-|=DY}&N`qOxo=>w{Cmm5B?sf%xW?lo5LvYB#Nei2wbBB-a|`>7k7mg7wJ z3qYjF!;3o}W3#8Ug8HyliT`CU;DB>r$$~yfOY=f(f{7BY@^m^xuZBSCo7LE*eH+1w z-QM8r`$#bFK_AGusDdj@(g)X1&BPO1a{1LC>gqx|3UNxOlPYSNhu7!d#}s5Nf%|k@ zN=4%d{Gw5Z{kHH0ho`PVm6OY0(|a!vmr)^65_Q0CvmCyep(A{aD&eHM)mS4aqRxCD zhc@aM0RNw@us(;6QQrasP)29glSF% zoqo1Rq11zN8F@>Y^o`of_`YUGrX3PY`I*eBm8WsM>Q>>BwK=T)?R4oS!^J4f^FK-2i=Pr>lNJ;*&kd;Do{KE0&61w`kEn(F zWh7mrKVZ9!6jS-I8a&NRK~Af$!Nm9lAQ|_jdhYbV8pn0~B^kajAiNp61uesd(@s-Q zdkoQcGe@94_zjNqDD#s~)xhb$4#Vd!c0r_KjN;FW(Y&LRVS-4EwmpgGZ=5}zGOb{d z#MTOMkC&noE6-Cyj}@VBD8@^7u)v?~ct%}cx*P7EeF9IFHNf{7rNB(hf7s}wNw_OM ziBeyi2b!s2u=r{sCAXf&^z4H1JsGPZPsSXV@tMebV|^JEe{2Mk#3ul`RD?$vZp16# zDzI`|Djt4MAG=?5g}>y-XMECUOBlF12T##CkIjAP1Df@mshZCoF3j_s&@8cxO$fzFqY_=t!FVm#<6uMv+UE-Ecl~9fU!C9v58H6to9B^!QJ3F#8Ly z5}u8IE_~9k9^W|I5!TstLVQyT{^Gka-l8=Cbxd|)tE1g;?O$P(=3WW-;4lSOkXk~t z{W+#oS%Ay`D})BOXW}l07;3%bEqM1>0+#4r1ZV#Q;ew0B_(atuU{c{(Tq)!RHn>_z z!WT~@_>%ch$GZxzj%eX8h;;)2pZ%#vBP9NIdIx3weiO{pYQiO>F1W#pEwEAr1Lu;= z@F&gVDaQT+C~DQkO+3P>-)tw=v2Y9iH7FB0e_o6)j7`Dz+y=jHd(KxMpV?{ z=TKAJVa^FKZs&M-oD0GZ)5n3M>p97xRoY-sbss43noME3PFQ5;GcbSpJ4#}Y0Ouhu z87!OwZatU-Y`sju!nX|2+4vc3N&kkKA5g`s1Jd|9Ybxr#1@FScKQ>aSzWR98VSp*G zx{a+$P@%3@bO|RtG-V=hTL^HRY>~rya!Q;rC+M9dl=_*H%yx>| zDf^n`J&GnxcPlc&Q`5LCziwtr=}ckY%)LU*<8Q$X+e_5O=HMtQ2C#vpW-Iz)s{8cuMRrr5;~50{N_J4sPObyb01sSCasbYH zkAdFF$+(qIKc=H#hsPY9hlQ-^l^pq!hldV6pq7T);4+23dCHGy>}IQ&GK{VTqZ90@ zTH}|Ld$2Q}HoFzHkI}B=<5X;M-#K8Dm@B!FA^@*-P65H$eo0}@VyxBu4d6+%p~t#2jZpN{?oiGRc4ORstiGnT>c{`x6#8@CCpG`Rt) ze50xU+xgh?XCmC|IS=-4QN<(DA|!WU3GhfQ0)h64z?IX-zoon47IFZnaRGS#nkiW0 z;XVBK^V@KV?Ks%8b_1R)%E11G*@87~Lh5;*4Vi4XkvXgsBP7eE> zz8MCjQTb=Kx_vg6F)T-h#_ku|w>)EaoJ{95C92%y;+dpROg{7LLKqh_*OkkjDd1Yi zTBX9T`#2eeC}F5&0-5MrPOd}E?APGfY*&3vrJTAv0pv=@Lrc4ux_0e zuKE;FzVDmB(Y=R2Txkh-zMsUGFG|1%cG-YQ&N;YbDvPC`FO-;e{=iietl=_V7QS?2 zBR^|!I(U4|n^ON^fUC|frz-9)f-5SD$9AvN@y0c5gCAP^8gDse1eH5-@WJ!SV2#5v;9@?TYQHOsY7)z_i##!)hT@UyuHTTH?G7#u z{YQm;AA*|6Z}}#Pv2beAEr@-dkFD9BN3|T~q5Kegu;hmla>B~$41o%(bWPaP%K*j$cW?U%uN;K;cx*MdB#yTx+bV>h6qdSIs)H*P(nwP zRlwnY=cs=hw;-`W4Csr{L{%eR*wR-Vyt_Xcu8OOKBiDsUpcadI7Meh-$!REH^aN$J zz>nLVwqXG$iT1$JGDp*6grF0)j~tmSmg|Yk88p^ z4Jyd-%_n}{vUHfeVi+D;46q@M5~}U6Gn#)i7{qqYKuZ>6Q?hXgwsloN%Yq`7<`xCyB*bgR>ijtlD=mZtatGkOgGUGwrgj28iebW!}35#VI zwzq}9vRWD2w9?9NMBTA%f>SFO2pq<7%y)SKbN|>8ft^gDfO?@TS-fKpA9uIIg3k&0)Q0U? z*EV^q=f+}8W=}HD@F-I^QE1MeX4=d@={KK$*-u4saeX}BWa5)LJ7+^|o75INTK1{V z&EZj96!(i4W>F(CQ$8VCRJfjBfYh+&=Pvw)AC3|Sy#)Tm{^$IwFXm#?9m@FYi$3u& z_d#Ok9p~Ry zv#oQCyULq0_`A+yY7hUQk~Wrlkgj`HzSC|NCtC-PwMkx;Ye_=m5AcsTxAKF^GkHn= z59@lLALRGg&E_8|l*d|EoaD=dR`KoT#7VN|lz_(c1gbSi2j23ViU+s^f+_`V$P1d! z-&&swN;0QQH06~*!0#|{u6`orxx^O}S6l_ezuT18C}fRwi5nB1-bP{6-d_ryjX>sqiHAJ_076%D_EDc>hy*&iw~V%r(s=$O{aKN8AV zZXkqH1{}!qchgz#BlCo}4$Nd6b{!V_B}>_=UHjO^gk<4h?G)j}zyfx{aO@xE(8enZ-l=}6sh)UO~A4V_xXo9B z+qMEMFTNOzF4dOs%%^}Ep{Zb~L!Ih77miIq4?#Eo5B0Z=0(F4q<*wWWc62QPE%yoV z1%D1qH!0!ybT77ag#sQ$&y)C7kMc{+lJT18K`MBR=Up3d1be@)1@m3*$lD$<9_gvK zV5Q5}04K#Xl;J9iR6qpqXlJPa2N|T>C5wskB4M@cE$E)P6`R;}lDgw3LM1Z=K-Y8} zYT*p|lk!hP!KxE5)i)8w@Qu-09S?Lez!A>ezZ$h_y^#z=&XY`Dd=pLc)CVWIXtZxz zF?C^_Is9G%uwV_0OU*na7z&JMRcso$p7nNQ_dp?A=d)iJVWQ9NwN@lwt`w4Ckuu!c z&O~;N_6uRbU>dnIcO1Fi)PnO8R0(ISY+EX1PqoiSGowv`*L?(<%gT{|vjV#Q=PGdj zC`;AfnvQPvx?seg3^-wi98#6j1*dLYr%s6WA?dL_z^+~sxtTR$-jWVzHsu)H9#RMg z{GE`gG-ZqpwukJ`CwkV;2L~%Nk&*KS$pZyFI8}E!*7(*KEfBKU z;A?+iQeJ|hUl}8h@&2HCL>0ab8;8-gwB0g4O-{g zpv5v~)N02T*jX%rAybNB%!oZ&t{aAqKU@Q$%Rc0m_(F0fJ5Dm=@;g-e%MiS}H^#7K z$iWM}^I-vJjj6ow1g6G!B@e8UsV=iu06&@loSF_$+x{NLR(@Rxq7LUvF7>~|T8-y} zms1Z(s>&bm%a`N<)j74)tDRZk{&ATx4rmBlHs#$|y^sZkrJF(A`#g|-qXx6$OF{1V z54_yc1Cm9W)_8wm1eNo=7ud*#VKrelG20+z>X&)0&}4TTXr7fISf?l?TRyk2o4U>l zT5T-YMkZ5OrD@H&9PDOiX`~62me&bRVbj?))xX$mellDCS}H6KDP{KAK41qPL)LL~ z2U{w#A+zlsvgutzf(duO2`9YCV}zD@?C$M=lxcCdKQ1H7VI_YVs~}5Z*Sn>{5tSUU zv%#C%m;8yMJ}BciVp71eMg^GQ6oQ!yXM#yq_arX0|FDF!+knV0gc^BvlgsDO^YM-Ux0Il|Ot4Yuh11i`-dQs$56KS7*(2IDipW4pdq3!*aB1(og-nML=L zS;~fGK4$j`s$#8}p)3vd>pUa2xkH1s_1h%;o#W3A*$M=?@zKJ7^{bf$!#|mf!Xv`j zf*At;0C#qbfu$Yx#tVirz6%@(bFgB+BGqi{O_jb}1|&W=uzi91D6x$iw&Y(gX7xZr zk}`D;_Ao62lX@3RWIQ|h*vmjr;e%0)QW`rMHAfzRc!(iew~mk=(VE)tQ^7i zIV#}eBr}RrScDz=cMdZ?^sP?Q&>E91bH}cW=Ss>2QhtBWA~5UiC&`KzZ?VV>ElKUI zd@S!3jRoxb$fv$s!@{R61-||97^$=oRJIDSx63Hr;dK(=((1*h=X-)NBCD>be-&@vmU5~ohmt#Di_SQ|zi8o`>b^9$i&7CUT&Z~#W z?Z-m7M{ni|wG0ACJufYGWc*}q*KiqW60A-y-L!;L_#(q5=r0h4oGTHU+_J$Fm&(Gj zCx@YMUIHGe`~g^{Rl;pT0#grB!9&YUs5?udKe93m81Jt?zKNX;FH1gv zog+q4S zlli$PKVk(n#+3Zg3Aj+JjOzcf4BiOegJ(U^#5X<@!HrXtfbW5)peN(Lq+;VzFzy8d z?#~OOcK02{Lc##98E6TwI#0k&WIpmBR30J)}woqw!YHv)KA`@_^5`pk94k3|*B!^LJ0rz`hhV!^{-{FvX$> zn=@f2b$QJ!D8a(`g>P?Djt3{fr)H)6N+XFxW#2Ja@Hhv{^<4{1CMDOYnmwYd?|q`w zTW(Q9Hg3?{_5ciSE2sW#p<&z8kCK9aQv3P3`Y5DX4bvNKfz#fXOCF;<%7$*^zuA+< zR=boF{sG6CqMv%)z0`c6hU`OTv|yaj2*k44l@o;NGgx7X%_26uhGI7FR~DYk{Ui+D z>?bU~a)9+zo+)_Pa!zO?Qz4AdsSsXmAIDuuYZdOCahnM%>|>uDtP&*bIV?;LGT;Vg zWYauBw{TZcuprM;h8-$i&Yql+1G`r&##&~6#u{h!z$b$BkeaIqN~{U$Sz{EuxhIF8 zrSO6hdkJ87s28S@)GbMgx(3%vu3?Cbg6reH@h=;XP~S5jQw0}ZQl*X|u=&~8uBQD2 zm96^{nzaT|&E<)Fn_UDl5o=+SEuX?0YtngL3VW!<{hHX;DO+H}HIAQf<^V=5?1T5i za$)B7Jgn(XBDLw>TU`P)4j1jPfhvf4uZw;=Jx8yp8HH{*ptmIw3$3vRf8L zSE#`Sv)@sNcP)jg^(UdWhBbY94c)g{Q9te;psZiq z^&N*X{@JiKZV>XcQn9Nt4O9lT z5@CEFP&aidl36g3|K`9L8@j9>?o}^@twR9izK=lLmw3Z=^+fc!PKDAcxGO1|^$u-Z zX9XfwW}^MGODJu^6*?a=!+uX)$R){01Uw>}y?pc`_hxW5d0LfXkvL1ZYqcF`V>gLh zknBYkf1Jn_+2^tjB3aUCQ4txo-;A6WCgx7YjP3WrlF9byKvFq=9yz%D9%nw9PqrH+ zu{pO6b3ThE2se&jKzjJ}ax3U`!KR%#q{o*fLbFRp*n#t%EGr+5_N9w~Uu6dvmX)IZ zd_B~&@i`zTYEl^~P6&yYU}v39!|pC!v{KC++$UaBy+;qD%8y=r zum=xFbhxJ-`;XlT=E|}tMco0N4U7fiBP4tJ`r#}!OpgUL_rp=Zhy%;MW4;647U zWFi#=R(+I$1=Sd}a#Aejv}g*RUS}a)4U3rJ+GFUpdf{uTD-2f%$=|3P@-$3meyUCZ`lM{#%M{;*FxCzHW4SDBGw zckVLZf;*=?lY7161eqDKg5x$W6UMDyP1@ZWWdx^nxr;7UNZt7Vim>WTMKDxRCz-Iz6nhX81J2;7l+%MhSm?U@ zlG@{%Kqlff#(ThEf1G#74*40=1*gl<&-480b3*KJlc~@Vy@FzHu0J>#6|!kGL~$r0RRy zxG6)1%!M+ZVNZv1_S$DiLPAKAB&noAq9RQym5_=^$doD3AYVCqZ_Y8KQe;FdIkuSjVnBi5u;=F6er z)WP~|>2ou2$&>E0<<{476gaMRae-!U$z+j<~26(?qj7NuM0{LMI zj5LShKFw#rshXv#Gq9cH!V%)^_q)$Uk!D zif5o$wg;bjID=9Exnf$n5_I2tR5_?thHEM50G)A;tneyDJIKK2wwjm1m2#+by%kR=RzAtxL; zhU@Y@qCxEnxyLT8ZPKPe#Bx=nnKWB|s7OpwuDK)#4g!q4oIAzpsb zqN3c6fr?*S$Sur$;Kd#UC^cAv%lE$lg>W5Wvr`-%{zrv?{#0%~I5J&BliGb~0dZ0G1em!*hQe2|@CT~dVE$4WF?GlQ zyi<(_zS-mAvi;NV636ZXF_5J;t6l_9P5;DX@eHI>83Kxx!Hi1jK@-_YPUGWWi@$cPn9^v zWRB$KHWz{XvVFpi(WYusK!Z!K&k0Af<%HQa{eq%3KPASS&j>XW1XX^^^d!TUvDIhRs0l0T zu1mJ5$_o|kP@%{qS1|L4S#_)(N1&;4l#&#r;$dSmfZvEJojxmw#Q*Aoex@6qx-pF6 zPU>S0+U z)jfZq`0lE+lsM@bZ}i0~ve%eFf6W}j8Glo#%(ppWYr{YiPMt@}q3fxs8%Ocgx&!#D zuS&E^Xcei@bsg{MSzS3JeGz4UN!{w1Aw@#7=26#^uJFc>a7pNSH1$MEj(S4fRKRUrR4 zt|bp{AjNaf{RJPmrKG3)aw^h%3!!_Z7u30}rDUL$_%`uNfNyvsUL|%0Pp?z~@Be~H z-%UCAgK=}>V67pgp)mrYT2;sgKhJ^A7COKD19s3o3%se^{QKdX>(rcp+H=46@cR*-Y`9oTSHnMzD;!gao^BQ{z1 zQYIZg!BzRsq~0?z$T*w`b`HjX2-Wui?sb5;b?`A>Br5}U@(oFQ7j3#_dPlv?+ZWgy9Hj-^8T8|? zLEtM=OYTUVq7E9#;ko!0Dzm?tYDwCT&+X45$2_d*URw@m^;V}_{yLF;T4QLCN4ioK!*EP`G>pE|Q>)(viJ? z`1|=WqQAd@@JH!hiQ9lwa%}ZdT4u8$ursR!w#T#SzkX`8L)mTM9j8oM#+lM53T^O$ zvg6d(hzTn2h8A%4XeNE557C_=J3#X#Et>nZ72kd42IaRWpZYud%=ACVrB&|6)5g|I zsL;SPT35D{oGg~(O$|)Yra$L_Jv&d)_LnWiADyO1%>z#OZ+!-BcKIx>ZQ%uErYPh6d6ap#fYx{!L^H0sP#Ong=%O~5+#7aBoJtJP zr8FCKxSpV`hw{m*!yZ)9$EEn?aTn=@zVib6-ZV)~<5Ow&0cTMwB9V;fWeAmzSV=qL zv_u}&exh@sdeXt7Y{{btS<&I)0?}}qg($nmR@&oyS>QhBnCLqiAUd{inMi-7S$y}OTWeGF_-;ufGS^Azs`gB%`L94HP zidVmTM-@+9p~&6!)DL5S`mSv@J-No7Dp9yd!hX zlJWZ4y#R4<7Ek*=!1LReradkj$al60I7C_y+((*}{LX2rlh2s=O;iF%dlLtIdwfBz zu`FR@2oquUx}bWI9kEW;2H(uB=JnfLCiH*4CbOIz3C5kd_>3A8#9zNMG ziLO!-McEitbB1iC7rG&7XYDL$ATL|gQ@K$(HRUL*3-TAq2{j}mPZ`p=4d+ES9XhN2 z@Vulu-h8c|OkXa1ja?8zxf~*9z9RL%nSXL$95Ltn2jD#S8kIUj%-iuvoyc2eMm`qp z0wT>maA&guB@=xGZ)^7_+{~6!FW(FUG_9G8nk)t^yk( zBM<1#H~N^bu0*uHA6O*6nQ%|z;u#-Xc@4|%5ye_FsPQvV?OJsVfJy-XaW%pi4&3rO98G_jGd27Y4U zZIaw_KwR_uEvfw^OB~ryBKGT;i68n@Ta6v6t=b|rRe6SVxvyTd!tB3PZ^Va*{`jh zov0+YB&CSA7f5(}TZg!97M5J3&l$gWMp0aGL658sxeGQ*C+3*g=r zTZbMc;}i769hs`Uz8i%2;!cd`>ve_vw%Ljdn{l4^_;muWvhJU_d?{nv-(idAAG|5v zzjiYjHs&u5w${bHhZv-Dft#4C(M)PP=#j5`5mIM^243O9qwDO4aM`W#;J~YkG?g-s zUbgTw&~_gpO;?-Khqf8uDhINt*oU%orfdrSMXrU+5O1Z6^a8?>7wrH`7o{U-G!&6n=Hl}vjzr&AH`1=IP(L9un392rb$R~0{V2LTl%eVNOl9xU8R zE%5ipTW9o8Dt}Mm(;f@psIZ7G!0u26p$?!$elNK${4QnFb%WRM!l&c{a;RNqYw$Fi zVDf*NF`c=x;H7mJmEog{D|kmx<9~NjlY1R0XvYWYeX$;W{=6zx^&X=$0_T&F;|y_} zZ#MmPUk4tXLeP^ndSqG7KT^KW1h1`CkzU3e1RCYOlF7j&=>^r9A{nRClCc~cVO#vD z#PNQmP~nt;Xywsn$@}H&CFMJ7g(r@#7sVcK5pI{tOM70P=SL`VMedbGBIb~~Xy=k- zY1@(|BL87)iN@$M>1>gA?ky%C^;hxI{cg~5%VmLT{hI0h zNKYki`#17dv@Kqc@q^O-br|180ifr$gFdYzPoFOo0P@>I@$LMFr?5KUp%jgJ} zF^LV}Vr6>?{b$Dzu9F-{3-qs%Jj{r?JDuIw`N4`{G!K#X6-TKz%gcx|9TOsXb0Wq0 ztO`6E%!xg9bIC_r3P8brIYR1ioP5c=iPvr3M%dPdQm=hDgy&{O-mb6rfU?)LC(QaP zSRCp|WESoxUi{kte0xt3KhGHBchsKnW;pyM0@hkmMT;{DWs4fTNHG{pOgNF}4y_SA zD5|VJ@n4cK|I?7@TJ36SrWGZ;=vXEhxo#_J2{Dm|HSLg=#aM{EmuCt8yEC0F?J1MW z9p+27?eq|}{GB1851y754~I*u4g^T|&G;z#&|WN^EJzkEHa{8?wYR%;lo23<`vibz@?y3RhtL_2dv7IonFCk>C zyg^zNP5f$lf~QV&R(?`6;*Sx_DW{wY;&Sgt9yN0{=(qDHvFQ$t_|s3xf>Q@6QNvxL z`^r+nc|3|*_rMVxd5aMuhCZ1vUIoMka|!m7oup1mH=bN^n8+$frBrsWAsRBW@TTq| zuxjl~V7j&mxUKOe^dpmrN23|wp{R&h_i~K4BBh^)$?5PFvTUXSUo;_;1cP6xF(5Q@ zCHd0!Bk;B;BjY-4DKF+m;@`A?JbA#0>WNRm@7X;DiR)j8kDPG@fsR)Iqt%TJaXX2d zKQEO9D}NjSWo15PQ>N9z2Aa*Fw)p?*c_s_bLJ$&Je-^Vj_n zjz8~_Xv7&w@tV%*reJSj4@^m{er=RCzyBp+c+L<7c7#@Y8{110kDE$^7i&uq?3^f1 z(MP)evPg*6dWmv2wN@YZ(2;@{`JxWeyLz{uht%P*oJ8%#THzaMq40T*2@(HVl@eY! zOL@$VC$?N{16J=!s0iUX-sPu?1oM><`3c$rlJg&djkd$&iilD?a=?RVyyHi?-FpWd z<%-G3h*H4#y$6&%iUD_~3<-~2KE!B=D$p?4N)#*)$9H=y73(^*5gYF5Q8`aG5i*w< z+-}ofxaqD%q-De}aG^evL|iPXJcH@Jg(ttj?J*Jc^4&GucjEv!$d>9u?FPX#g8evVGrJZU@kaExY936x~bpCEI_Jn z=5#ipiHb^T<>kuY)K|L_D#xCU(>ue-U%Qm)A2P~d2zoq`h{N9((&-&@-wiGTxsf9k(_X zEdA&q8unQv)Nu8cEQRAGzxV6YT~WPw{@NIjesMQVJbO#sTbv3^FUyenIx_UceKWk+ z-J9ZfcTtU;m$+hS3@Mv#L%;Z84RVu*C>Jj$d`Ze_>OE%-Rhqw(($7$*%XclI&z7lD zA)ox{k+Q91hoqc$r|Kr%!BGc6M(gQdyT#=B1{vz4iU=nU4NzFqX1r}&1%yf>X#-1H z+T*Ssn7!c@xx2WZIw9!b751;CWJB*$9T(*B#=+fWu)ZFRXmSAQHj~zjk0GmfB~k%F zhbZULXbRe;Kxdn=Xe$(@dY0JJo^#d3H(#Vwjs#P**5qIOm+MA)%2bZ}dY4BnX*I{A zo|}{Uyi2^pMThVOpprc7w*a5r?Sz;7+JLXgY!~;aw^jCJzPYyb=wk608pAtZ?=SW| zHzp3LzR&%)Hjk&faw&e;cOH4{c&qrqLS1o@cMTc!Erax)6!4rU9mG@n;(4Yk2gGeI zrDTAm5a-T)#tq!!$(xu%^Xg(giW8C|_?5C0zh0}oIg`7c1bg4#s2*rn6(|n72)4YPLq* zRYw(M^GCkq@SnF|7o04Ot~NDL5yW`s^4|t@3&Q&HtCeN1gZI711i{Huf(wV@$&-QA zyn#K5c5z6o8-ovYzz7tjLvT>c#d3pGjlAUb6Yd3$E9+7rPPrk2h|^BI|H@itCny&%H8B zEd3+q4$;NDTWjpe;OJ=5_n4SB-d2PMs{JC1>wa<%_lR(%;kkH0_IdF;k4KgE1`U<< z)t2IKYcqMN8fU}?e^}&)V=BDLL$uhV(g~kAV?9~pWk<$)nu-;)`bnRf&t#Upk=Sj` zY*I$9h{sHwLB_S4QgqHTJVJ&=t~ABX?g7p)Us>A>9vGBhNYuu|>2Ks^_1Hl{p5XRjwPc9al9` z)(nbJI;BipuHVPsqOqJ$f1bhD(U`#>dY?+z+8Oap{#EdUF&zOpKZ{_g*%N4Z9kG1X z3;r$qF;T6f&-c%%B`EE0#ERJUe4UH|K38QuUuYl6Kgc>vXhcmC&yN=pF-v0jo6n!* z!((kk*&;*!c(y#>c6AW{@cTsmkLE1IOdCcsyxg&}i$}3D?ys@hquFRwSR5)7!p7En zsi1C^W@r)nKlFRwf5^>?<8T}N5pxo+Mi1aOF`cQ0=n3W#B=uA<@-f^HJ&e=nFGG1O zB)kR{0eehGDG>eH6@WHJs&J02I*(eX-Nj~U&BZR88DX=3xuQ!_jj`_EMdLspx@Xds4b|+Zm-iu_bA>%)}NotF_3Cuiji8JfRg#x z@&I*o!A=9zEmHy04%NW6d$2j&g^}2e_yFu->>TX2@o{v~!%@z_LSsyQYCY%pV-L)+ z-;i@+;22tTP67L4(g^R2J_eWGF~E@9)mUAQHoQI67wvU9hm?F<1xJ@Fp^lqJk=hfl z*!71xVRP+73_EiTrl8eW-NFS(>whL_dtePJ{&NzR`?LV9%kjZ(zQ2oV+2vzDm^SF6 zIWyoWz6WO_Xe08w>mhb$LI*|7&%#WZ9mw)EbNNS}R{_rwdx9rwCYtJV_&MHQK=J4h zaSB}l{$>3j=9XFTh10&o(`$V}comCS_$r%!{Z1P3R8@|jm>vfb&#fo$rbPbfAMV7N zXB9-JM>P@0GC({s9>WJ+N$fX=ha4?V z#>U6!cqOXSo{fypJI_(mx`8rn4LD)! ztLOpQgGhSdFC@XM6KnCVLY7m5=))a7*nXXZoKzGYZpo7lfvAo^g2c$I()``5c3_Rao-Bb7+y@HPlCOJvRO#AC2BW59?Ohhdr|~;9Ll} zgXY@FV;=&_P$**xqI7xa*^gJzYx6019cG3)o-yL|BTLZmpi7v(>0@*bHH_>Y>qnEN z{~@s_o+8547qQvv=VO_Y9~fa&ztLPSD`{FO48NbpHJ<)%J z;zxU8Lzx;s_MifPonZ!l$(|3yo9qnYir;18P4zCKhrO8p-ToIiGzb%`mOmi&k6z^K z*S{d_*72r0jB<%lqi7;p?9NZgdPa=>Q{@ML`%Xyoq(riuH~(qH24Ym=#vj|chJQ_U zB@y&poo~PV1(81^hd6rDlphrCgQ+@QL?>^o$F3`yVvn8!&ccF^sQXq~?8glUEd5Cp zyhPWAu zELtgp^Hwm4ZhhH^wGC%tKaS0T-HyIPM@<%Dhn5v0(trf?mv|+|?MDe3Wa7!`9b1o8 z@A!=5zi&geB}2%huo6*`Avh0*YO$7e^Eq`IEOfAX){2^%(0WxyvZ-QQrRz$t+tI!9&yD<+XGt89D;TY_Z z#kT9NK+B?HIjb|5W34uc9Cwprn1TX?%Eyf%d*c1zqsv<01dS_*=6^@gr}O?IANHJw zZAS>Cv?LbUl6xMClv|C6cbq~z{qo^0hW?P&7zuY=nGZX>H$c5eYb2_x9s0M-4OVU6 zfSBITL4xbkpdTM0B<;E?Vl(mr;u|R-6+uiiPtXCcTdIeiVy7UF2K$je(-iO?a03@a zUB!dORR|%s7ihe!1cm;aKq@T*WTx3;*g_XDdUFJP9$g3C>}P_F)14TfWWvGgy}Q8k zeHkG2+cog`)@)!s)(m2k3<(?ahhWomUb&>&8vK-T$6G4hak14iaN#S4ADvO`6qnHQVryIhC7caxXk4%u=w+0b=*}>JQJNoDRa=0$(HBzxi3fCyyL2Naa!hPQ#Bjf8D5Vxuvlsvlz)!5XCzWY4` z^`AU|m?hdGYt9{j+PWb``Nt7-#-KH_7`_6%IPeK!%E%zvOa&Bfn1M z`Hwfos z3i9;64!RQhiM+p&1a)h|Fu%nMZU|?W5^aA&PLSi49-oZXxOy3eMAE+1LKSK}Ri@9k{R+1Lmk6deRT z>2p8`-xO@HivqtVPlI_G8-Xxi1&pb+fFySXFl>|vd5FWjOsB2LvCcR|!Q~GG?S2N8poZuNJOYM(tD^hn??<-b@^C};FuP^H zBl2UJ8{%X^BIWl|;hEdNAh$=;p)}1{c-A8}s%f?oDzbQpENyH?s*Ntd#%G>EA72ln zwqLAKt(_lHXxBLUh#7*tbv%zo3hdC4=sSqFiaz}EwkuMpRDrgdFGt2~FyzbBEX=;^ z3o`i70xPqAg0^YZ!+I_f#9?_pB6@lei3~NuoCIn}-iD8;*+o-iaq|z@+ggAzR&>HQ zb>uJupQTvm*Cgay);+{wV>}wYAOqn_($HHfW$3r=G4xHpDLQt~8}0nviC*tn!S3h} zL)(Lnqw(+8B7^Jepj;{#4LR}?8S|(>zqZGqvbVCVc{abYi&&&6~!c>X=gUE2!GS^|@0TxexNi*z z%E<+#&<;Xdsguy|Ga|+l--AK<0$_>u;cmr|gzo`mVwXcISi$llu*OD$(vTs}|7ayr zRIKkx3AHq25(XSs72V$o>KHd(o6L6#YgDp&@0dusO;YHYjPy$`09*30?U1(PEQcm9NUUY2R9dy+!HdfF0 z0a-5a#*VYg(GQ0m(S-SiNKa}hR`PHjW>fzZ`(|pA^zM%U2iVH)&@=M`JeZ( zTat5Ffx#n`I{gA^J1IsV@|SU5dM-jg?kdFwulr%=_nyV}FjdiFeM=+w9z|@HbG-@S74p)IOs#I4tt<+6ynGph1*ujK+T;y*v`R^V4@-e{-J*kxuv}e za;!6ef4-4HW_~wCTOhC#`9El z@!w|aKwh;Qey{Zc?)^alH@`i=D-S)w+ZMMJb<2*1R*mVv_lR@I-egPE%K<FwuON-t6oLE>{$)x{!NFg3&Pn|zh`2`d(2@l zLySoO-a?RHmYB?*3V7$JA-Y8AFygs#4O^RW7BPkIBCo{b@c3;lcvJ2a((|zq5$QXj z-!>*7nqPjw?HfKJr=Pq9K4L2;Yzy}06%??3F4o}Y~M-yPptW6m8%?_2Hc@w$3 zW+URyeT`%fzJ}IG87ObC5@Ebqk6J&Sg?Zi>MD4dNMjHYFvg!L=Cl0s0b`%7smJ%nf`2pQMbBX_N{!7w| zSma!K1*CPa0gk9|M?Nt?IGlVpCW8_ ze*q#p3qdyJ`@u>ZQsCSA2iXyyW#M)`efUcDdt|Gc7GiV38d-I(j9sVQ51HDiqd`Y5 zv%gLlBYUeiBd$M_VF#ftjBneBeCa%i3}3K9q1|Sf$$y#{SI!fCO#Vby_t1#>*XNLC z-YoR*f=VQUc^y8yrxPmv*M|P*+73P2e-_L&Ozf5s4W%C-?Q)vqQ+vkq1ytf*y{gsZmtCavc*9U-UaLmeWR|$3H z%S5o=O2YZX1Yvva5ODRlPb3oa`EXuPsY6I#9V9|xQwfirX6$TV%G!GPLYKA1Xd*f-22Bf$3VBpsg#O!N+^f zVJojhqixG}V7&=RXjvr(O}$=%_^edNeBbt=5vy#m#{)H3^r<&kXMrl_IWG_E);^2w zU0DQA?np)Fzeh2+|3Bnc{9CyA9*>hVUW4SPXJGGQjIq(PZ;|LiJFLZL5b;*Nh0bKW zz>d|9qEmJ@9J~OajqOsT;${>lugw(=(9q*pMUdE-Y9Q*uT7r5UJI-=XSi&-iiDMFJ zqmcjJL^vSuCM#2v1O=~DfmzL-7N#=+ba{U%>!=%GKP>#h#BNu@IfbRn6&M3vZPCIO zMHsVntmi-v(nRQY$tkuwGX{2vX0W%b zXu$ic(%21G#a0#580Md&H@JtU`$zP~%Xk94+gwq|d|vmt`MfF9EX(4lMZEkKv$$I< zAl})|M&4u1Ie7htF5ao9cX$Ss20Yoz9absa6yBUlU0&{9GrXh2)+)f|C{Lrogj@H= zh|6dx;e7_?y#5o`+_?AsJmUss-jbMN-twAe-o*Vhc0_a+_^;tHl;5i?-#se7MUzT=Ckg=ixqFMode`qdem_w zk1B#xCuSn528nQB>{F=C%K|>wX@UH1ZiRynA4axMDj?K5S0u~z0_45JnU!TrpsZCo z@QWQO$cfp5@GMPxq(O!cz5H^CrOD!1jQ?E?Iqy}074NFBH9a4gIWBt1e7c5aUHo|6 zwAE-S)Hm@8s(jvP+`xZpGNV!qZNDbRx_842GB`!Bd*pYTjSG`l^9p7`oex!5T+29< z;NA5mPb^%akjxXz-oY-j+LBHtFx5A$TKA4^(_hN83{PaGd_|c2+16HDVzfCAV8Ba? ztKnJx^RW8ORl#lsIa%fgX<7A5Ia$vAe1wNF2wvcmA@;(dvz9?_4!oLVn8zs?;ts5< zwbF8p!onvQyy|&4FLZmb)uA~jA>jcR*mY|hx9zc`<@YXk%t;AY>dC){MwiXv%D-k< zJ+GQ)<(uDNW%pr-t^Hbq>EWyaci3qo5AGX7k;m^?{5;aka+F?X55G1r{q5}n!}jIy z)Vmob?Rl3NXH=8n;pAnIW#KjWx%n*Ub!?+WSmg!iUX(1{{HljF^685yT;FEq9q$I) zKPqEgaMd=Gy<@^&a&D%X!^uC;%J$vt@9)jn1#PVi>gihK-OgB+q7eok_;VSKEFELD zhSWhTPbVRI1_#-v>wJv!_w+-5dkm2DeV1TApyO*yu1WIWR4z7yKPm1kOP zQ-wSev)Km>U0~5!O&A-w3EwP!fN1&qL)nebnKy5pXE)y30acC_!^U?*VApOHM47L` zjJ$c%G(%2>@os%7+c3YBad=Y<>xH15ablf6(=J$sHS6jEhFU7Y;GK$P30}Ttc3=v| z{0E85XH*dL9YfvR=CvAohw^ns*OOw4Op8iJJ8f^GdsUNFeNmTjFHDD7|KA?wMwMdY znglUxCj_xJKAXeboiAWI9w=kUHdL8&mkALKjZ7>4ZX=#>N37M~!V0VTaoXIMoh6ob zsdKDe&9dj7_-17F*tyl}#eonj>6ZnT7C#0pSML#Uod_!|>7^sLVuuX(`^qftsypRY zue>z5{*x0{n&0nOf*gV6&Fvai8=kSP@*0)73#SULu)o%pZ-1L|zqb9bs&JogrBnCb zY>s*(%R>7sgzQdZWjmZ>2e7@_ONLWfCNDm*ls76{yjL%0{+Nnp(_7+Lvz515ByFD0 zI;`r-;u+~M;Q-R9&Y6 zMJ}{s_fFxgZ+{N3L-NYmFWY{YebAZ(p;r{4{eRM-|2||v_`}^0yAx+G3VzR?-0sH? zF{@-OT3*dee&q!j%>Q8Vymc>=u|f-ezxO2j($;sZOORS8Y?4RLj8-xg$Qanw(w@D0Q3dhq24j5D?tt& zS4cE9&xl|Pp2f1vY>rR&Cj8>c-}SUwY+O!sR{^kPOCc_(OU50;Re6tReBiB7t_9jv z3g9nC1s}fNz-yXjD}Dbr3iOno0OC|L-sN}=7(9N1)3G1G>baqWW|aW1voglN>0ZPi zJE!18xGboNok6@Z@x%XXn?am+vqlE9k@ZSqM7(zs z+OCM4s;>o_|FX1Qp*;&Ul*WVONQa2R_V~?9tKldu_+8xg7-3ITzm4Nmct%Tc5PNB!Ox~AbF1F(JLJjmJ( zMs|)oh9CY8MgL1$h@`$-3|}%AAbYO)Ep=S3wKC0lRCd8BD(Oht}Gxf+Ay%LzT}SLCG4GP~%d2Ug0`FeAzN3Uc#L` z-1Z{}%<-b}$4BSj7={DMs0vtHyB#=EI=owY0IoPy!t8 z$O8^jeQX7fS+E1ifYf4Nu)kgsB73Zl!?Ulvf>hIHAe%cCpwnnn;>r#}%$)$Idr{rn891*o6=5%q@8tTt=KK*Pwa!3Vsa*H^nJO-3yZI;f7eIM|JLZ4iPO#)YHSQ*u^8nRV!c>a zvLD1K{#|Kdwt2JZ37wOygP}3Z%78fb!SJ%_(!Hn51M;#=QM@H{>pe5}g*+LRx#UEdeTY~i*cXVJha+2a)G;;t!8N>k08SEPuzgDm#vO;m+~IE>sx)X zOtreDn`-HFuHH&1V->f_(ukX}A;EI236I(DVQu!!Djilnl*roE zxYz8+H^A&&)C}2Qk!7$dirA}L`0R833)#OPLrjaLXl7aNGH8d)31(5f7n_W_#`INK z!3x6)Ep|UnVjq9=oqcc~#2Rq!V;9`jV^&7bWC&v~v)*i&!E*kg3(0;x!E*a+!k%5! z%Us(j%cy!e$tq48WjRkZu+kLEScA_vOiv31=GVzKc*dF)tmc2-a886dw7XA)_@s;3 zhC9+=BW(&^oH$K>N&61@*qwl8GCQE3kqOYQ^{3&b5$Vvbe+jTvR}!qVcOmO@KMJ!} z{bixHfoz*MXJPGM@^GEiAbhmL3@#c|g}#Mfg{S2vT<9ML-{zV^>eU;e>GP7^du|bK z4u$Xp+w<`Gpo_Pb{TbI>(#JdRZH@1J?1rmoX5gCM7=C!cF5cD$DmZIhEN)Z141cW_ zihIAw!MppCa3@a>UUldYKJ&Pkdt>!_{KO>~-!f{4KMZ5zaz1=~tMCVY#c&}m56R+d zG-lziV^8qB3=!TUxf(q6T?8U{ZMs(*HAQQ5un5B{q z2-<87W8NR23%&O2zL>YLqLKk@d4!GZub6)K(i~)o46E7hNG98`aTOF*tbmw$Na0bR=+O5nR}?209!(iLmbZ!+s$ni0|Vpr0>{WcF#&P zn6v&heBDh4miJUfJ%>NQvtFG+<)d_wf$^_U$g)~A^Me&)uek$VpF9Rva8!`e!V>85 zz8O$MY76rr;lgq{EeivkTdb&%EJn+mLT0>QF3Tr3%lx^O4|_%LI#!BaE^|+F-aM-B zIApZ@9K$toA^ZA&ajc9eCC1!zMJ91n#IpP+Vp=mk}Y<`y>`;gi!mc{N}++v^OR(?L(R>Qixc&o8*mIB#Q%R<2iPIm5n z%c%E8TtR>{x63ivlBFfKG?V*-`fN&Pism3c}dp!q*mV`!Oh;p3RQ!nawJ=y3+h_y%W6Ad6Hd| zR15hvdb0gCh?(zF-m;GNE3hNcT4rIX8f>`B4!ZM97kO3sntAI1o545BgZJ)_U^=wW zP}!yDkU+1LeOKuZb63j%tQ*K>^&=0VJI5U1J;O&?Sp{pM{TPDWV0~e=Pg$5dx*da_ z4^_dJa&lO{E3UEWu28uC>S0#kthq>IgeqjSDw9#$SIUkW6R}MzzCceN~ zA;{{zvQSlc0#hc`iM=y(1~jzB0k-Z`gSp`aaQHt1<|o^y93{>z%xL{g%awAemOuYo zu#BuThwX4b8t~R=dIBWY|y6>JsD<4Cq7Ns(;-Ip;kiX3j`LkrWc`b}f{o z(k4Z_5E4R?C`*eivVOSAHUmbRJq>M zXkTW8k;vt=QKjxYP2@zBD>B>(jK7RlM=0xnqp<#|9e}uktk}d%TUBeASIL+f$6L+zT;ko^Zw} z_F6md{_zOcC1_Kd4V&pM=`5lNWRROqU#LQu>0+px3#ErBj90 zlzlv_eVN*JY~;5sa^SZcm2kZ?g7_QvE^{AePNynqQB=}8kN$LE_?xk@ypdKue=DI< z&o;t@4^kdUk46M>p962}6-*dMFSd&FQ{G!q)$JeoNeQi7Y`Zk4WvI_PG-mLsdmH(s za!S0zVFSuL-QnH_1#!0z4pGNLjM1J3S4LzTDp1dN8FtkvEsL2X! zzx!cgTayZpZsvnkqZ&EWSH{f#83vjXx8U2|xmbOIDfZdZK+5|Fl;2-KB5fi7g=WL| za|v+E-UQZWog)T)SBUXL6YvasOSbIG#@7#z#TT~cvW>rkp(#?FJwLP)rs%JRLrxml z^p8KSSy0FiIVcEED}#i!a*9IL_7!yY1UqWHDwwu<*3-f3NLVFHX@`jvz1&$#o3dPm z6&98Bt?zwW>+M1-`u?L9QPRTGt~UBIIa+8`krt->>ft?0HHEtYHGExMJ{<^%6Yf~d zr+1rG1^Wj}Xn~3qW&eDl!%Ln?FTWj}fA1rMxdE)Fb_1ETU!7zu8AmqD2N4sK2&6jm zB9u5GlKtrhR+lWo*#;tZW9|u9Hk3q;&AkULQxr*#Spgi84}pY@eZ=s>B(~&AB&=LY z*~PlbAeML#|Mx^(Co}dl&jv{f(eWYl)4Wf-tNkmUc_l;FzgS3fuAZf_uS@xH zK1cX*p7ZFe-T&y*elr^M!jM`RKI7kR-b+^m#?v8Pce>7Hv=HKdj@qX$;_pdcq%L0< z@n8PVri_z{;8oJZi+xI@W#7B_bm0d7-{&wo*Fr>YrO82C%^#?l^M!PLG$b$7(&1@e zBFsHM054l4k^FiVcGMq%sB{~==?RZ@$EK3TunDl^l{k6(U>3Y^jDWx=I&gX6XPlGi zK_U#&$TIy;*mq4aN&J?C_08wu%O``_FOlmZxBN1hIK~G4gpG%UE>rw^<$DkwKgLR1 zE`vgKH#t9%B|WQbfz4bCk#jD=eAA0pa!^|en6K3PV^?hhpumU@3 z%V)gc&NgzYTpZ+^M?&KIe<=I%Z(gg*iFewYLyPl!>4KCi^o!+f{_zn>T6JR#y?Vlx z4xM{Lr_0{w=hmFy(~2BvIg%129|OJJ7((Zph||=T9rRIW8htnF1l3>mh1RZ!pz3ue z`8D&e(%V^ad{N?b`oKn6*ybWmJHF-9z{OHDd4UuiHz}2pZ+%df+>9G5x?oLr71@!p zo5(~igVssmkURf9s7;=Y^dF3ay4%qZ>%9oSoY{$->%z&t_Cf3+(kB)2)4`uwK*08P zeE!mFJmZQ!u~ghh?y2emn|6W(4QpzXr*6at{GC{h$F5MGTtuw?nL}^vED($H!+#^s z!l;ec@y#cfVc)T-PrPJ;M-%vV~3yEpy zN9-m(j(iK52{$&41#VnFR%nxeQy$|;yO$1`>N*m7b2pPGIxcvntri~s9k2syhVbtP zC&*~ga^Rn78vhKC(9fmRr23F$#v1@zmlL( z>khea9JLPgp)?Cma&vn*{^*tuUN`D+T<~u2>oEkMNzd`Y@jFP#`T(L{7lt!ieMn0l z!B3wXBl(pMY@XUUi1iF7w`V-T3fk$IyP$=31wQA$7pw4R38hsBl!Orr5mirp!3UYl z;P+HN;NP~+rFJ@+^v$&heo3hVU*4-vgGnw`h+a$WuZ^bHI7@!{jaa&9?Q$x)?=ycg z&{zmDh@-Chh@ZP_8NF_p!)@xALesY^2!Cho=1*6xqZc0^=evm=?|n&;HcmAqk=Cj( z)@%SWwi>XBmhz-w|2>Ec-U7?C#*?N4$C$KyQ-J+7aPpNQ4wPGu4X0iqk{%RP?hnA0 zOjURzJcmXf7bqRngdzWO5;g4s={KpvHO&`^+R+GHwI?6VTbIsudj*30kLzT$n>u`- z)q&4gUt?}wlqI_~2e4Fi1vDzN?D@J8)-0AOrna zcK&pIICT6fc~?0TR7SqQZc|jzQ%@IS`O^rb;;f0X;s(}dWf_q_a0ULYX@W^@i%6W_ zNEFWIK;2UyQ*k{s9HYY0+zE$h7$Ba+GtI_Gu8tQ^G7iZJtfiL|2bymXrfQv#z z_B`R^h9h+U^Al9%_hq`+U0RqPxr2lpFs3gLhzqeD$-+_cjdqTf7gVFuX}2pD z#@^%wLtP!gWalN};wwYp*GhFdW91qluXhw(8`DIkhH3=be4NTnb{4*>M^lr;1iIS> z2@{^z5jOA?JaS1PFFdnY>!~tqFKnWKb$|o;FuOpi=Z(z;5CFE!B5H?v=hZ9?! z*-GPTn2~swxW@Lw4)v)-zvdxKIB)@0r%YlSe^|0DS4!Z~rVXrewJj_!%){%OSe>}k z@nn==AbThEJ%pCDf!UTbSgA9LJpT~{?XCW#ry`ZT95_HG?CAojQ?g{m+->CHi(uq! z)dsc41mYN#jUDAr;FdICjhizdZqF4mYhF7T<;xIXn@{ks^&FImevwC&6Ird@r(t%N zB^w$p51VGK#{mOlb#(X|I6Gql+tVdRBJJg|!zu^l7%fOLnF|$z#-nml}Mh2y!>0ef()85i#|3^<8SCtIPY$+b${uG}*UW?W4 z?!)U=KZ307-@&)*FuwU)jf9(xzzg!Op|eGuq$;r#_vo9D`ko?O?wXIYiwY5PzryEA zDDzi5y{WfV6?Lm~qZfXk)rZI{*FQ zWt%nV-kEv4u9^uA)>uLP>MW^h-wi4`)`^BcH{_pv38GF&kM}sGNaK83sGxq1-|%At zJ^Z|oUniZ-FH%@cR|VdN)j@gqg2x4DdUcj)29F^YC9^@{w-xZ)is6!H7n3pY3Kw2; zf@#{S_@{RRGLl_P6n@>nV**~AcBn1F+%?4vj&)qlmSE?_*hAsAk?gy5Kp-C)1^4B43T2fq3K!Z9kU z&~+;e6kB|lO-Vm+^tnY4xBC`q@XkbDp(^C$>_VKr^%nS@{D!j|CE@y?d-(B<+t{@B z7yLDsCF7)XaoduaWbR~T+`Fv@eR`t6TG>CsT-GM?EK3rik9Ofj=@(F;*I2SDFSu}LG`@+NmP3UQfB{=&OSsXbJg09~rb7!2#k5+5qEI7{Y@$-cv%3lcIvjqHS zZUK|$fNkIS(NZ|ae{@q3X7p|q%qMFI?k`Mfv+4?J;}JvGpQxtq_RbJ=e+zU=)o8l6 z{T_|=TqOklOr`$x9i8*SpGL>_(Lo6*0q^@qy+!_lF7t;j4jW0etXScb#dE&#U>ddc zjuvDVt)*gXv;@#zK=(XdPmd?})2&M)iGPP9IB-71f5b{Q@XTxS)$k8wh>4N!{ei@N zaWpd3PKK^+N9tO}Wh^@{%2Md#n;r*vDyg$~A1iqdE^93W~;pWKB z&Z;N5E7PH+w-SaT4T$Z210;+{hIl*(jk@93%Gnav|7#BM1DyUWQAx zCm=Q78_K>El7e#&$mrcc@aWnI*53FbmJOVXPbb`AJ3XS{>OR8WtGEb{{-i?ZH6G)f z{~&$mV?6cKNK$ZPEbH1IO_sYl!3L>t@EALaJl~hb+*fpgP`M)5p=yqEUWnn3LIyeA zE)F>(mXY*&Ke()a5bnpgL)v%@#36zd?290ioa|tue+ePy60pW%ar~k9Fxyk;3O3#! z$blVZpz_~Z7+J1>cPp);Qv;Lu>#n1P!P{$v(2Z)suKyUSy3m%k<%iPJ_)Mz(7lb7< zc^Y_0oQhQz&?>=B7{4lo-Y|Jf_XKXH-wIyS>E(ZEg3L#1QsOO0eRxaVHjkvIA50WZ z9=*vY`i9bv(ILWV-zC(pV3bSWciuRuUy4nU-Ug4Y@`&?}Vwhz58raPTz}emp zG=dw5^07~(wJa3|l$6*3iIK4Q)fU|D^o(t=tOwVL`fU5=IJh`23nCu5;O1A9n2YPc z_#2emezchNcv(wQ#D`IJP$`JD>yip92APcxgRaj*5NN*}J2va!)vfmkGs+SAtdEi` zlXEcpasj+GNraU5i{bN~3&i5Xb+Tq-Fs#4dPo@Zu@dtH;b01c-X};kgKb~P_XeP`_ zihy>7vDoj!@b+VmSE66hxd{IbVm@cxLWw`mG3qb8(4jarRNyEQ6>FI?;ps0JRXaUq zzrzykvkTWC^_zdRRm1hsJ!yaRQv9U$M(4AljZ+In9*?exzRq)C2Il+{od_*P#?h)w zYfGm{S9$@OWS_~%Nfw~Kw=0;Skq&6{p*E3b{vF*rO=@~^-XrK7g>jWr0{BZ*#2tVCQt$poO>W15N-jb^ST`vZaba4?+>~%>y#RTlXYkmV z|Mx6GFaLO{?lj56xbLkn*;7|;c zb@-fULu(b%u8U+g(;sN)Tr8t-AQKfW3Pai}+C*li6PdNneT;*wF|*-iv9>X~iHvX5 zY5$%x6@B(Pj*OL?v>!H16ixq{B#KLTBf39*1G8Nu&-9xySU30pqZJ~?c(2YuIZsqj z&MR5G`#8(A-t|Hgv^<&R=3AMVuu+WLxM#@n$YZ8%=o!+ms9}76dZII0Cg{Mnt)dIR zmNPw8@~9%*n>ny;pLX@pP?R-ZMbzzKiNsbVqrmsGL~1|ZiC$)jGqQfunMc-dnAJ&o zjP#v0WVUP!V|c-Y8C6z*oXXOep{y5Z{KIX`oNZocq|qBuh>|VTdk}opR36f9>>vTH z%ixv5Wn9#>6))awJp3JKi`14K!hchm@b}XS=z#b-bVBnDEZJ*_J;qvsS;I73v$`LL zis$2Rb64Sk9Y4W)cQ3@eR>nRm?eNbo1G$?tFx@VXNq^cAToNHmezniXJ6~#Go4wi0 zCoOM&?uuki&*3b8q9Bf5_SmqWjib%VH2bIP^`!9ql51c})>g37Xoky@nTO7#!a>CR5wBTh- z2mbF-GL8_b5YyufNind;yPP#iReB#Po%Mn7>L1HqKb4OkZCyx8$DhYhHS@67?*c|r zNkklsMnQ$m7!ouGv#;91NydjLxO*i5T%L?2s6D`cVzDyW> zMJPXSNjmTCfK5edu;*kL?2tvUqw6Fg2MS4Qtv<|qbC2l86ye%J6|AeD&Iaw;1NCBG z$P2j*5Z1Z`wolc@rc(~m`AJ1Q=PV=WzuPJt#iND0rk3(f-hB}zy<=0I`(Sm8(CQfjebI(<3)HPz3EC*3MrLGj2sVuv@d@m6=prhGXP z;5rDt_jeHAnoY=wIRVpx#*#5<<@m$6Q#f&-GP~z_D10Zeq+9bM{Kyzdax9KP&`fX8 z)crtehkd?6>4R`5Rg3M2Qh|oQF*q)#mzDVV26}XOw&d9b@NrCpNgobkjav@n`8+#t zOcqFl%L=wKr;Kd>T?)#7Dq*h2G!k-X656xm6oh(w2dNutaq0>$yl2K|ab6b>|abK-53D3alv$vpt zl0%Hy^bi=|_7zS46#|Aoci@0d4ZP#y1UzTc1njYl!mY#8p!51(JkBBx)H5z1KIkZt z81G6ha3n5LwfcM2le$YUq^dWx`7r67eAdlj zy=VG0URiM)rA7Yy-3U!SZG1I9Jt35q`$h3$|Hf1MxES8I_7$&@u1QbFQmUs}8 z@#GXgl00eIjct$Jz)L1Ch2DglOoRJ6{KZHLGK?P~xwmahMcM;UyJd?nKB$Aswt2XI zlNXK*6T^!y+TuZtAow;S0lw>1<5h<~Lbba*{#-AIdmo=6^4ava#; zo|%AS-dw}_c}iH$RUN*ys1n(>MtDA95jK5r3J))chS1NhjPmDb+_Fm=miA?!w3^e* zt7S>x-8u~qhhpiE$n4!MbR%*X z>2_U(6&=OM$-TvR-K|8DVn@pnr@4cU0yQ&ureP(o*w$!;l-Fr?&Stg_UgHBmEpxBb@&gB zHC)F;f3CgVn|ps*k4q1$<%H5v{IROvdWtO<^dzpb+`TP+eAvQ=+!Z?9i?p7~HC~J4 z4s9ysW|7m}_1#vy$+oXtc%54!GjNLZmyw2G3di1uw~%jO+dDk$r6kTD~@_f+ytvRSVGac<-n}l@l|7A`MnBu;yT3kLigSqlT3gk=mk^O&1nT^LI zA#Tk@G<2{Gx3;`Q6SrufUH@HUF1T#KD8UQ88f}1Gr5do;w!?UZ!b3Exs~Qz1T|_=s zMWS2E!N^oR2^G|8GgeO9n7QT(V6B>n)O7x!!b{iCoq^RT%dr*p-}}QT-R?xW%kE*L z&l}L>o6_*u$PBGtw~H}y2!lJO`_ReYGx$fc9xhR}L*s}EGZftdp$AW6->Hv4!n>a2 znav})c@!2@Spe?84(p(IidKd9liaiR2~gqF~l zxstl;C(z+)0{t=WEUi$frO&<`p$2C}Lf5})>iy^v?|i$5ws;L2T>rVy`d!{FCInvGSAUhT@K`_lOojQ=#Uw56WEDtDtV+-04YoJ!1as{39ZsZp`#tZ z?!hg%^ZgL^oH`pnk$Xi}ZlUny%oF_X_Uj{9}@5PY2`|2QjV!e)wz@wumYueu5$d@=J(mg*SY^atKZj=R}r1`G}2P zZY0h=7s!|xJJ2e6NY~#(`{^SU!)Pz(>222s`@Q%}l z_}vm$bkra5#$GAh?xmmj-UWKptm`2^U^JTVIWUda^fu>}bT{!%$Fyluv@+jedV`Dh z+{k}dol8Sx*7HM~BKfzwr|>QjVf;LUCf=dx1t0CGO7-(E^MOUDI8Tca^!A)n++Cd= z{Iw=yTDDu3N8euYiziv~_m1oHaz5Yr?)S>W{7ee_w2XzsZ%wdZ%0bX?+lag5%5l&| zNiaCHi<#*Yino+rz#E4)qfK|E(7|EN=1!)Y!HMzpu^1aY>eF!0$6Nn3@3z`yH{zeG2Vg58|E4 z81Inw!)4~GcuV9DT=UBZyli<~WpNws{?J2O+1pTJ<_>a0@)26@`VPE`(sA888+9Ht=q_?$jGSQNRS3P3*9m!yHyaV=(*?}FNEW!h$ zev#mWIq*tx2U$9t6EVoS5AWApf=MOI;i^?V`KtDoO!sgG*{h@2#H)94*=Y} zV15dEwrR8GuOi^zr3mm@ZiMq*pQTZ5jr?pm10mVvobd1MWFb*CkUoyvOU=&bQP;q3 zI?KvN;EmmAgAYqTZ~jc%)D8(n^WV^x)UQ;k;|P^HG*K9(q%Cl}MhP=Cas`QCPVgOO zxp&{q7s5w>6o^H(h_vgUB{QuP(a^*D@b(VXR6X!2N-_#Off3^eOpXZ6f z>IZPIGX-+1$Fco+rfh|A8dQ9CW0yo2fYZHGxSrF}sSlPT&QCV7b_ec4-il$~p&NwD zdJmKD+&YMN^&p=I6Il((XGF>E8RY8z1Hb1h$?h!{=yl|<_r6&~kk(P`Wpe_*y{60# z9Y}_zhFsE_eH}VvrAh1aE-(o4g66RTCk?1(O?xVk}0NHf$vegvd*1ozfY!O><<$d~BZFn7UT5?!(f zY9e!Bw5u_&C1#+1B!$SQTqX`a>mbwj9ntym9_OWnV42D+w)<8J>=-G<`p*sk_I4OF z1qETfj_2P!ZArHQvG)#Dl6Fxkw@>t<|lKpm&!;S zzAufKH0y!J@1;a--vP)+`@qN48g|KAzyY;Dq97JQ(w9zy=1rG~%*HeLnwCDc(~4xV z+#FamoN3>ACk75&-Urz`$6)Wu$N1Ho)9C&A<5;Ke6NEf^j7E!wr(E8Z$FCos!7Bpy zYxnv;K*`G%;{x3y%#%JTL@iw*sVxDm=~Tl9?=+)xeh2a7(KB#R;d%7$b^shV*$)e9 z;uybwt6{-HHPl#_%eXMsH`4RRfq)joiSi z|3dsTr>k7^dyd*=NAhQK!*!Q8&*nW{!t@qiw;R@gpYok6ow$)|lKi)~d$?^|-sx>q z9M3yXHi6zO6+Cx}7{ur4kni~&=*znIc*dpe_}1%xxGgXCg*>7to;(i< z#%WElFsL>LHcmf`C-~e7SK=6+Xh4{zMM(w}v zf1xn1t;1ZbnhBq@6urm_0=HwW$RarsD_xYwGhUp>tW_|U`cs1b3~PLYvBePJXo2R+ zB*K|796C54k2Ks45%uiJ_~wk`@O0NG9R7VT3R+&xa7{zFHM|C05G%xAtbV`~hR1VP z9>x)6CfMipEiCseQrk^a9LHtZ;o^lwj7g9hI$;$Bx|yZuRk;aPn)N= zK^5BO6AE9IqTsdC7sm8sG&of-h})XLXv-}pe*#!s;JF80wKbqJcW>nX{1_9y`ZJfE zC*q#1HsI?E#!}1u0o+STJ>Gv|FgI&@Bj;_skas&gihpSA#;M%)<7B#R`4#22c_V*! zK4Rr+?&bc)+^x}v`4OAUdHnhc_dec=)~2NK`$CrLN!t7HI(AF-n)U?oVu2&6$aWt$ zR^E(%^1+*vX<tS72Uo{1zBvZ-OgLRl)ha5$PBihVvIGLA-rAK6VS>n@_yRvSK0D zw=Kt=Im^+sEhkX#@(=I^X5(`0!`O4{4m|$(5XRe&;P_k>JTc-X)J*GxGoAV<& zGmL_8{O`wDChDiQag`2UaWoSDZtY?` zg8WQVAyoY-82nd*Sc3*Qkj*0ZZPkp)>;mF(+6|u+p2GUKviP@k6gv2R6XO?a0g;0U zN7hI}1l~?=hM2?QVScz<(+~TJcu-uRBYLG8jFpU z?@q=`Z2R!O!Cbs&i5s3UwHNl!>W0A9zvx!vGf=;B3N=U;GrMlyBm1Kd<1JT5khe2z zaLQ*TELj@Q%=pjxQ<@paKl&O&6G~6>lQ!D$?UxfcwN*3t?pIg%aY1|epN0Po zSHQ}+5zkNYwSQIU&(;opm2xbv;&PfBaZ84l+^gq%+AR57$tBc1UxuD1sT{wsiqFZE zC0`o3pAoP5hIbSBvY`x4DKwBj)KLhT>k@G3Fm}2Ax{x#u#X*_3IkdfQ z#1q!V!{VnsBJ#Tor!AWfA$%&*j&DL_JWo=(b8+hSeXzx5KMvfm2yVZq#l~@`@f0&7 zV*h&%xe#xT3o=Dyr2AhKT13!Vq{@EN{(|S#n3B?=FqBfcj&hj|pb4+a9);Krlc zpy-oHp8Pln_cEU0oR~~3v(*6%-iL|4KF-H)6UV^cGDXBOCz0)23F3V{68jw73YnEo zc-D%EpgQdtX4nKQe?*B4hN_XIi!4?j&WGfP-k|a|PllTZGOTl27H)2tPM)jf;z^dX zu}CL@G3qGC$;VHj?6b*uy^SInHCq-tw$H^5>s!&o(^s+cWLZ(ryNAg4!cP2Y^KnMS zN*vvdUkVu#naK2kJ5KpjfV7?_;;9`Ru9ba*Hr-4Dcb5aOSF?f{b!`zeCRw0CkME4Z zeS0#~mBC|LHbdrvb!ag2Fp@qs$XIlja&65ToKk?m2X8?1QkWy>Df-F9X>Q>HqN_Nc z(eAu@#!PrHR{+{kba8y)43MOOx^WAz2Ll zBjMytSFEo^@gk2Na31Cn+MUwKCO#F^SDipEt;>d zA3yg(#Pv=f&CarK4tb4~#AeRz4=Ix^|RA=1#(k0l?T zAuGH#Vy{EbP+RFXHj14C5?RGWTuvYQgES!U$ZK?8gQvIt`t$RDJfd0SrwQUwwNxan zN1y*up+|Tl`t!&!+E?|Jy8i9wXQ11BXJ;Js$X5|cVh_+o;t{l(V`#cVG4(L6pc4mg zfejDLNY7kd_LF8L>3`S<{a4>ZmZSj*aG#GB`fP@PTd!csr#KwVg<%`gPJF&D1&QZ2 zq>()hUspClt6~t~Gdgf(QZCu|ww);KnFjx^{3Z$skMOOBGw|B-a@IJg5EN2Iu(pvQ zV4>*>M&Bo3z_v{F`g&qc7XBFu4J4DX#Ib`== z9;O+-Bo5qJd_-mfR@r)r70GRf!14XWa>p9D@?!<8C}8kbKAb)nlgArXjTai!whF)7 z6@}C3j?~^@BOQ|*MyDOGp@F5Gz-(aY`UY7#La&Zq7+N8$I(nLRHC?BrkNxS`gCFSH zixPs(jxOq>=PTsBlokHQzT{5?=m}4jKjtMxX;dXTTJYBNq%-5x1by4NG^)vguC(i+ zz4FIN%FpGXJKXuXZSBB%|EVI=hQ06;UlqvBj&&p>@-X^&BM~g?C5d};C0?=dIQGvQ zH{8PxhM{GLiO;)RV7B-h3~Ww;)@i`+&DV69 zikxu$Tr#y&))FpH;e_TkRY7UN1z}I6uJBqzo4&Dh6$0){(*+xvsQ!z~g6FVx{O`u zZw+7DO0idhmd>e?3FIg1$C|Y?!?Mtq;F*|)pC1V)6ii^$_FxO-rKCX0 zqe0^Sb{w1WD;+YnOk?M@$iuonTd`ZoC>_rwHL&H6JL^628zk#i!_#0Vyl3khXu45? zo#UHf_M9gqMA4dDG_;2IM^?j%r4QlJ2RCMiB@ZdlJD~rr0$v^W3jH*XCW;s9@x40= zWT&qg{HU>k1@}xK>P9CHKWs-_Grh^CY-vdNn@GNPdSW%D*5PfOuB=+L4mhsJBS)8O z!>WmzFe-is#rw~s3UM3wri)GVeKi&;Cp6M9tV9=QPNel6^V>yd$mS zQ`SUL(_)dZ@5Oq`4jrSur&xMKBZrRpQ%^@uI!VKI^o8}V*Qu_4F<&6vOQ%N;`$G>l z(|{MY!lU3x)E2#@rZ2})y>bN_@g<+$c;!O&jn{w|T9iE5tHwUNeUi*cs)G$3cVHwl zmHbH&P&@d;xZtNCOpL~N0_?HHg?A)N#tOvWdJ>O0ad2ZWA9Uxhg1uTi^qxOX(o61; zp&yP=oBwOLGvA1})>`1szzeL*uvWIhY$Q9iI0*E`*29{QhIssVM7*}iLC8lrvdUCt zI4>Db)&zzC?|TeJ{rd)cbw4mNDyyJ=<3(_5nT8drM0jj|F_G=mgV*Pm6033BAuBx+ zN_U41=aE!^lif+m`p%Qda6sdfQd7I&Zi>F$mgL_ZnpHr2Q?b2p+^3ON4^5qH9ue+tpt-yF>aZVh`_wd>o zPebvTeR-lUZ;i0ky?*qN&k^a{9AeIW)LxwwHJ3+k8l1u#SoqZfS>A?Hh z$Z??y?(%jMu5%$qi}~{Sc>eiOMgCHrCogT-&P^Vz$Y0xE%1`;yg`Pf;K>cSf;S0ZW zQDOTnJbCkC6!pRcZw_CGzs&JwX8K=bp7#7kPTup7W#2JT_epWs`SpqDO8gFdeB}(h z<+U}_!t7!$+$W4);{;TuB!&-2g`zkwHTW$Ti&i}AMfYVtz}=iP4d0E@Hn z3xZ(HK5aJe^dEfd>T=w4=PUcypbEPGnXn3VXP`?w8#H=1VvF=D+V3&Mi~OC0pw?@G zW{$J)tRs1L+n6J34B$g0L$!OPD=PLO9p1EHqmd(#M5U zg}@{;LE*lh5V-iZkaTjM@FSm4*_K^GptvfPa_FY(4H|{@#@UpPTPG|CI7W*~lj#^o zUf9C55f|t4&_2JAv}TmDU9-lqz18+4*l{7r`&2_bbZ#T#S>M5E(s~j+e>7Y;*NK_# z4Q$M!`(SR`O7hMskQx39NR0jmQ22Hg0_W+oe;wV})z7NIe{LWf@zVp2B~;;`GfZ@r zuI5NW{$ci|pJ7paKJL&vP0oLh9M*pik*e%GHla<1jj@&@3kuaqynZ;*O9?|R zD?6ZP?<_L1;u7AHbP=~zQ+B0PAynMEPc({tLfN@VMDwE<`R`6HTwXhxty1UNjA8tC zZLI@)_Miq_lnTO?^BJB04!wr&hC%F>y(7rMm3Kib&kM(_y9jTZ60m{CRY(-F$ms$r za%!C_aKcouA1Z?-rxq}U`U)_icpjWzITCw1W~0QDtA^V{sd#zD7*Z+{!Jg|Huy05S zJX~w=g))ZJ)EJUitz*E|Ya2;Xcfv#89-x&n2J91aMF^6PCo3(aVCQ`G;l5};aY4P3MID!GxP0itGaGWWmzaOp}I7_8R;4>A_A*7*_hwWo>cVlPnFc|h8C z)MDSR1vu+fEbI6?60(kdB$C?h5S!=+Z!N8Hx8XNnlV0O0`5zFlZ-6MRh$Qz4=D;|Q zwJ=v~ICmcVh!J_(g6f1c-~tSAgnlP#_l+S=Yh|GHo-KLfvJ8~sUBKeXT=;uQ5jxQV zGOcPKiBmEK&6np%mE(TwGR*0_wF22}ZGiWyo|EN{TOj@I0yw`*3fJgwA_>-3&^FhN zxUC6cJ^tJx^AFsHsQv9wW4?}f?w3VB8ZN_7^eEC+8jpKy*W&JJ%B*Y38W_wuL1uPV zLUYO&Fx!0(eD8(Bz}7D$?}QjDwkr(!Pcv*{?jQWl-32dVzO%}t45BX1WS5M{1gr9V zsNTB?@2bB>JJyNOtZo;g3t0I`gi*@;ZNQyq2GC`pyq!^sPncJmLQJ0dhZonuBy@z<33Q0 z2~P!y5t&qY>?3$*CefY!>9l0EzA*AgE9u>x12W}##5edV>$qk-tD$H~ZoHjAMDaI> z$fq9ZjQ^u-9j;6t$Js7r#IZl!>4PYGe?%gd7Mj?aLv+lVhT zJ^?SU8oZ3T3Z5!m#NAMteE&BaLere!S9~2DRNl?R`T{r@_(DDYj+)BfqIQRUBxG+h zCQG`Y`;#i%W!<2?X%^hi>&A=LFC|RYMxq;b1Ao{NLrVTzj(3=UMFuxr*~GI5eh+8t zAMIBLPF)t3FaL^;cxh1sn*ct`u9$W?=?UwK(rNCOQB>xR6fIUFbkgo1deZzMb$IZB zpBbIaTllZ1Q?$nm^^$%xKFNb>o*6;4b5Bvtrc7#76h}RCjD$U^eD7k-|hd-H!7c^1)65W(T#!FOl6X$E@JzvoQUS6Dmbt( zA9jU^h`6zU5*I9i1!cAH^5q8naM2Phrf{7E>6=0HMux~}FNX&OC14W10Q^)L@Ee&- z>`^tz{wE7z_RYlXR0aP1#R;3GrL#w3V?pNp2QpOU3a_k;U|Od&mUNdO%T)*Q{vR*E zeX}NeQp23=8o3GbJ|BWh&Y$3z!F?v*e>s%ADTal!$6)hM2z#AAM#6{thB+0gBqrS) zPP{n`-gG5&!G5KGqI`-ANll#vYWF#PSuXnf^nB-`-I0$L-RN!KMW(EQEA zK+qeMC0hgze-rViujTM}S0Qn?aU&&f6`}UhH1Kt(fH|A(nfJkicvwsaa}!0(Z%9R3 zPumi2`(!-MLyOFwDnQR!29j>cK|*gi{@lcpv?OC`1g`ZJDDf>z2Qr=cD$!$V^^)xxA zk4jbc@{@jD=WQP!rn8jgg}ANev|lx!?vI~Joy||v>YHV>f7W?wCsGn%bvm8q_=x|M z_L>@~#qdiun$n^1GleMys`T~MS5z%;3@zHKMfE)r>6kJgWpV%T{eKh4lvFWx^rjr* z)DQ!YZ>B&(93nMaw2?6ESB6iy1Yu?o_^#7f?6@@J-F`!{I8jhR05g~XTk@XtRfDLmQ%~dDy_C*887;j}OSxcC@_#yH6 z=naRaIYDydGgO%tfxcd?W)iiOaY7l3gUoEKZ?-^mB8MzL&$Z?6H(`e4LGj! zCo|8MLCt52vFh)6XolTrY`304kCPqIB(Et*b4n?9=GYXDN%rAa$78Nfr=y@;2b2-sJ0{Fb^;_u(i$X13>hK={Fx$WXRsr{+1Mfu zei$Ov-|8fjE07}Y4JRWxYs@2*S79SjBGV#Gs!<|4-asTwd2uA&57HrGjNT-wx<@1Z zv5_O2cp)Vk644`;b7>^4#;+q5GGEsG)xCNm=wl&m7wlAIxsfQ%&TASxn?M{6SjvyLMV6+R=`_6sB8 zB`YGhKrbXCyI>^Ol>s6L{F@{``WGX3_Dmxw0Uag8S7#&75>_M}qZA`dtY9Mucq}8u KtDru$GJQXn>e$`@ literal 0 HcmV?d00001 diff --git a/dsr/dsr/test/data/test_model.index b/dsr/dsr/test/data/test_model.index new file mode 100644 index 0000000000000000000000000000000000000000..9cc628d1a13471baa1dc44f4eaba3b7b204de455 GIT binary patch literal 564 zcmZQzVB=tvV&Y(AQ06bl&&f=#)Gx}*)Az~DOHC{aFG?&ZNG;M&NzE%x)lbSyEaqSo zVi98E;M7ntRrH<1smv;<@0gO93lmmQ0ZObmo^7weBp7c97xe&&HeLP~C(kC9om!Na zngf*K02-qJGD;ywqk+@Z_F8tfDyulmG=%IMPLTAcSq6MsOyXcOk>rFxa#tOs&q}I@ zAD^6>lcNtJ;tlk3ic4}K0#N6(3NdpuFetQst) z*-<7@TUwXYk=z5(R+Y7yv2Gh_oq7t;X~N z&_Yl5u)Bu@B!48?vgOEj6vy@|{;a+J#a0~Wvrg8r)}Pp3JMnt$#Gm+|<8R`Pv(6@a zPJD8_Uhb{$t9q}hUcL9a2QbdTOn1-gs(WwUf8DwTpL=)smG6c%sGsX}!o3jOJRWS2 z-?#90gug$6zmGov_~GpM)^@yh2<~bgxa&wzo87yoK0g=kgO^|yzk4~n3-)&V+dJbC zexngwj^>~~+SwvsKLEkyu-@&DPXs4}<8Mp`yBCf&!=P55-LrRYe*b}kcO4Fo+28g}*(-xVFRGoGIlch&QHDrVJ6T`b3$x8yeIEWifcbOV+r3-oI_<4+ zKh$4s4-s&L17{Bi4*cb-{N*9oejSVa-X3*(gFbmT53|>X?GC<-5CFslU*W9B7ZY;b14o*NGvqpcAMD>5#jW08^G3IS zO+)wq`9nV*jpO0q)#NYmJnTI;8pXqLco60{2S`amQlv0K(P@v{OqPwLT%=Fuuy5C-}bF?(M_`Eque4s3re? z9LdW$>v#lr;iFxgK5mV!x3^;nZ(u_)_$={KtIS3)xq=@vdUD>tHy-qS z2ka*_Xk8r)x7y=y?`S+kx<-4$*~`zobYbl-IKW=v&kN7Ig!Av`^FNL27aoQKh-K?a zd-KNjaIii7`Cov0&JPFe&SrZwCJ=Pv{`lPR+6YCnHady((-+GG5d36FKA|tjCnzd{ z{+d9%j5vugHwJ{RR0PRj2VoCg9WHnKh1`+Rl-0Bg+hv%&fS?k|x_NwdeYm+1;mcdc zXP+32iLRg@F(D=I$fx|>QJC8p?({d2LnLp^z_W1R!hn4E+|{e2c#L`okfZTEk^hA{ ziWviVp-!~YL4=n+N8f6ysfHS7jt%uC?v{n69^CQAp*n=~T z#?jr9etSAZ%#Pp9B_`ZEj{Bp*@Hi5j{)G_%PyypN$HPHyW7xfREgr%;9DL$dzrEGn zTpxGGo7WNaeOx(GBnjw*p>c%>1rYSDcz7))sz$>stutL@L}>zCcvq+ zJ_>Mk*xo`-gFl445Si_cTDTOWal1cG7VP*jII`UywtKy}*XrUFZK8WdLYn}b$Ejc0 zY43W6%5pS$Z#kY_|rBSwL$oIgAMv1%s$mca~C4S zIP`JAC5&&j2v4Yb%{uZiP`^QqwI(br=34c4Lv-@2e>q_I|_+9 zVMMFX6SCoo9(Z~fqu36e3!8Q`n5mn;>o7V6@FE;~25r#*jpWukH9@yfvDL@zYYY3( z<|M+36xzpZ)UAbm3bPgaWEQ@FAxK-&q6IObF=1PZ*%s@T}Zn%n7tUYZ)m<57X~NI9tXHM zxVApHI^JsEBt+^Bu63z{hI$9^&4~Sp@VjyHH`4*So9*7Yeka5!Z=#QYziur+!~h_| zhRzpGL!P=k81tvdpYbVp6U_6!JvX!t@J5)sI2dfBUhAO53U^QS>EIREd#OFXv_rK5 zQDuaLR|pBwjY309yh4HoYMt@0-5+fa(DP|4T1WfC73~kg1g;$bXwDIiC7vyOT>$k< zNH?n>q!jjJ%KZLFqe$Y%laj7eG&9(8Qk(?KS8VNKWh5F_Anh*i0 zaqml<0$qxmaPV?`bNsAuG<;kFj0-Ohp4%quvG=)t{0wTfgVLuuUfXzBhJUW>2y{_R=x``9X=&^s{b@5Miy-Q!l~n z^B7j7T~mDujlor!xm%sPwlD)1VE^;I_GWy2(Cfs*tS72Fp2v@JKRRUYbTqqGTL|Eb z;9g~rm%8KYPsCT-$Vvz$4;q;Jnmax|&%!?Ti)yXsQL*1CS}TUV&)<3q!w(WsQ2o-Z zZ*=<+1`13HnzI}Df;jNw!Hu{t9pNKDp3_5Vq$Yh@|CI3em&3sq(7haNy|3RT5hy{Q z1zc$B4ADZa$la&K4{Or1IDR-08J-9s^S5CS&X3BO1bXB4aLfWd^0wFA>MCzLaSwwE zmVS{$VDB+wl+Lxx!@lMobZC%|-wpd-h}*r=p6L6@_ZhFCqT*o^iXKv)QEKl7{jCdz z5}ZcBpO8|PWx z=pZd+#EYa;jHXL#!{Z2l+>19exg<0`#3duDn&y90>#is!Oh)Zlr}-Ym(ZXbrh&AU> zsAw3ZeCPOssC4Sr-D_Ket_+BzIWl=- z=u*B5_VaJhToF=+_-`aZ)5&$@@arg>>x9iTm56%tN_?%`KfdQ#;wT}?vm?yT@wZkP zxCi+2EZ`8S(8mxbQ@ns)c3!aXeuQqjhe0>y6i4I*Pr%@^3hSu24w#IXr)~v#64hby zhFV9OZA?TK-T|}guVB6sNk%-JJVIYaG+x<>W6V;jTxDTFj1muw*yAi|JWtb}n6Z8& zs=>^O0L}rdkJ0u%KiFn!;X2BEFJ3r`;y#Pqzjb_`{A7H37?L-PJBgfs5w1NQVrO(+ zAA`6gbc}RBnBY(1n$c_R?bcSjH5lGX!WS~~IwDI(#!(Nz9EN%=!T{)~q52YVqP9;1 zg=z^Msuv#f?eb8-_oUy^!^tD}vF;Z%s7Q zx{$%AwZ9eD;%CuO*yg;5{`+tFz3Vf8JO3=99qKqAA-Z`> zPMyq#;SuJfIxPVhCM4$QUy+at0meXkM}wUqDs8@zKxX)2Z3c9P(lgL32n~CSphkx7 zz)=NwwDn_*e6sM%1To=Np`P`*bUYGWL}qzRLtq`%O?HZeM96YXBayHEy8sR<5aRdb zpH+p0`-ih+1SDaR^+Tn0pu@7^R0}4W5`Bh4o4vR_Y_+cF+$8dA`9nI!DuC=c=S38@JEnhhm{@gF$17C=? zsg1iVjJwnby0M?fA7O26hP%lCiY=x2^ZVxN;mlev?+4z8;X!3gWk+y{ViD4G zM5NH8v5&-*1Y_XY223=N4=;z9ENI{sZ;A{UjjqjxwYdQPRsi>!CkxfR$S8wl1BwT@AQKIglo+U6ja1+BWU9_2gGOZJWDG+h!hZn^xL#NR2G%#mw*(2EJVpv{lrS zOCIMNj|ru5^8BN7K}dvj{`i|`WawZCW}MJ|eKmPdR|CjXPgo}&qUVik{YtEi^_y?Fty zz#Fpbk7gsy0Uj%D^C@6cGfx~TGDn&RiY>d&W?ur45s@4&0Cg0*Q}xFI95oG}%LmQK z0aoOuy{69%WNS=@2|#ic%Bt!wA;K8yEz#X2?)E8CY9&_V0ihMb^M|N!2#H$+-ygtX zg_3NJP}WFk%9i6C8A%^mKvmV7#EiTk{$kBnGLFCpG|jj@z$EFdrD4CX`@voW$ts^2 zh~cKS0XMZs-f|r)Oyi}C>l>H!Pm&_! zB~njp&Gcys+HU{;|0|z(Nu~f zvx&|L54;ba-;WLs~YIiaMa=SA>Jb92zTDam~J~R#hrgS9&5WRCT2k3JJ{+Wc_Vl!?*~a3E*-y z(MYs4`dyRI*MAbg-||Hy$!d5~K}q&)>;g-o7o(!qg7?WX zL-WpK1}u6^xK%UXlVb*o_U&T^xzXga;4W4rF2xsCV^91xnsC$#YUa*hpEV}5<~#mV z4W60|Jc=2goT6z)6A|WSG)bZ-d|1?~6J1ad5^Ab&^{JK;UVIgE{{6M83X_Axb7K=? zr>ZNa(aGC#i={PK|H5NC1+H<-QNva{5j5^IRj*+Qm2O-f%TS5yNINY>wvxW{ahyTLS|p zdakYU+^SbO2}yd;2#_4iy<)2ap!C#2g{+PabYpI@mGJklZG$;dB$IIG-$gyXFK($= zNFYs6%Wh!MmYL6E;4u4saROXEkG+zE;h(b8hysPl97o1TmYD{PyOa~G zr21oI+d*eX>G%QTC06&!xBJX-Ps*KlWISO9-ey(;hDnngs(IR!;+fdAIoY}b)BXo* zETEwqYUduc3Q%RODL>fu;kEJDnngt@UF6{=@%b~y8Od>rBk7}&`8(kCMdCT_&x* zSs9BH$5t07rlKg3H7U6-e%1)^V6zCG`)5zL2+N>2_#l~U1B_qkt-EVHjSX)@c6F_QP$G3%+nllCj z{Ykn#jVO>$Z<-LqvC!qSEhFy*DD119WVAjEeK)}6=h@Lj^h5`^TO_o zz|K9<)=M3kZ&=?;dY@IZ2-DC%q)=5#GP%2ukUD2m&lCanOd}sgiH|3+ z7<76Ad$tI$XC)*aPi`2Ceg9nyW();bzGljZx&RJF993)4SFCPjQnMo55Hb)-WKI%e$0|ydq1# z42hTx7HlTv3op(cb)7ir_5Euwft~XX;Z)>yH z#(UZta3Xvd?_ynhPd@uuoknsat&@ffAqxRw8^qP60?o~uRBTJ=N=Z6Yy_W!Gf2;mHt-Ssq#RMf_9|#7}u5UIFf^D}KtC zkxv&v{Inx?H_6&cx)mGE7yvqjK4yE|x|wJ)15T8|Y$>oHHX zrlw6FgWm{j{q9u%h9owGKj1W`+}}LuiI`-=Gtwb=2Vgy*uS0Osv0(qj6SKtA@xiR1 zt5a{3#H^gJ+SH>ERA!slz-=obK>Xt;Oy(@lw97GGN=kmT8IDc$=it& zg^ zDMc(wi$!v4wI4XA3W9U06ga+1bGjfjr+t>jck(j@CqJW-!onclOnJ88Ke3_YRrpxdt!U|(y+aacTz6Vi!WQ`IN2t)P+uZHS|^u! zqwyB|;6^;`$GxRz``vN3-NSZQ!zEhI$u5X%wMVTnz2H+EHGM(mJf4B4!Y63XqvGU^ zH;8S=qvu235xjPMZUygE`xyS$8@SjHAlKmk4&bGVuowuT?xHrPfqs?KyWdxb_f8yT z-h^cOOchOQxu&vC*@it2Wp>IA!v3{+ys2&_tj~RnmPyjB7x>y5JYI2~LglvD2JUH- z{Zw}QD`{DHR9v>I&99u@4;s19;yEq6$W-1OONN~Szr&~N@Ki-wES(#Ty4U*QUiLex zO514iLPSm$?Tq3UYRDVi{^#>l@4;9Zqa))a|B<|mI%Mq3>c+Dg*Uo7(U?-po-6|FrU` zM$Kr_{O430lV2ON+HmnZ{LcZLueJgyg;8w<@Y2TgJ=%(J3Y`nT7{I2h21)mu^>=w* zG>)6Pc_w;`v))d`a)!&uQysi)*_3}@1Lq7n0{S~Ybd+{EehIXuJBKK6Zk>vDV6!dceR^v{k-@gJ1hZ(j2p3ITStinj zky3wWtHsc)EzCAPl(C$b;bOMJ#8i9h_Z0lADT-3N(~&oU`|_O@30lMkl`s#G51a=K zkY^eg8-MI^__Z3me_GOnxZv59PG@@DKv#XO8BI;I(d9^(&C@SlB$ZoJ6M^;awXH!H z{RRFan&txfHlS1*ntjIVVFYHy!HjDmcZCx8>ds01ZECrK%|WsFkSC z98Mu!sZQkpLW-3%7l_sl3xC1XhQMP$>~74HyNAx-g~{DnYTjt^lkPFtJ-B)JdF<=y zF_&GA9D|b`?sTDKSzI%vQFjXo+Dq@4o`-L#H^-Wl)EY?QOcXXDT zHxxy|m%|B<6<$h7i4R>81RY!e|Fj0D{WE{ePhxx4vfyKCfiewBqj#1A&s4{umtvS| zJUlbT^YE25s05hU)-IqW3B0>gdy%75C1=4WYH+6L^6-j;q_(0Y|3&XAIWy-BO9_N8 z2P8!J|FD*|5-5TfyfuNGd-qtvr9-Hv&%|Tv)Ed5lg3#`(HiPQ$Q{QUviN`yPi|Kg4lPJ%qQM3MM?ju zUT)pt71iJe16VEyDT+%mj_&XM&y96VqLV%a2N1 z&rj!APnk=7$R)987u{Nv;*@VV|1%yqbj_pOLHE+IJC2vqAH28{hS|>|%MFH1EYi;# z24xxCY%f(rTa@RT4~c+U6miN!DNUdd|C@@D?}D- zaI{ekB}W;$A#kd25w1f&!w=Tr-L6$o@LnIcH*cuSwT=hZkzo(3&(M!1#jjOmsX#|$ z1qUmyVFSdog=mFalh~gw+S;(2m9aF zpjmv)!acZGc`c605QQ#xR8go(%}kf0B+q+w;sBW4=yq-rZy!Ea2WN9}g)cLTW!0`J?Ufc z;98`g>)Jo}y7(8scjNAcOA6%4u_f$YvQ2HR6+3)BrUTf!@QaOdee0>AUWbhWe zX02XtY-afBz3^O$Z*$&en$pjwzJQ6p*$qq?gm_4}xu;&Q2lWPSTaE`e;(iF>-1e=j zh;i#Ad^P~*RY!PzW$XH2yfo;yu<^o9T##9_6+Gk!gUY0V)&Bq80N&wR2?^TOs}qAw z7a%)q-x#=KX8~I$-X32sNHb0OSX>B0o^u|4DCeGb4pg}Jxqkf2VC?Qt$z)+|*&U>4 zc)G8I5swN^ii$0+PzFI`4r&|E(iGScm~G$ej>7#5LifJzodKJNZw=s45B_Q`mt}t2 z+Ia{Cd|OU>j#HBWcN^IMn*w;7AL+^nK-RAYRb$xGyI4i%)ZLP3g+w6drcOEN62&zt z$W!jDSTeX=X5SPqy=sYNh}{wcsA!$r1Gv&98$WodvIj&ZM0oxXc1D|7gG8F)Kjz%> z!(daoI5;oG;pB8%cu8}&g;%JO=q4Uba8cpk2xb0_*(_-NmiUZgSot` zwLffGzCAbTr%YxeKKx`2CPf^P^s_L7vQ&7Z)UGAv1bZE=L_T|Z z(MznGca~UVZT31QsH=-L;$`D3LH813-JFhCn}+oj>!z<*`!*0dn@hI}{Msxo)^_3) z5bLH%tQ`sSMuj{)FIBE0)=ezaJhI&$wtKy}*TNm7VSC)gN`4>#ySYNST@fGV8Piy^H1*u)Jky@@raj~`&r+`?mIOOE7 zm&C7lS?kq1ORQHXCDw?SWF;G~!GA64gs)CVtWCrEiuJ0mSbINs(nc>q(4d0guNIeU zJ9P@k^{PpDE0|=LrJsPgUUjG;_&Yh9GEZX-JsS8YU)jMr#iB8`E!CHey^m?BEcpHa zPPjr(6{_%n`Yzu&cW$+)L-|Pk=W@1-_=VEX4a0f)(pSjNMBX&;s@BTY;aGswx`QGv z3J$9}t`>1|yoAcJXG-&g;mMoZc$oN!?$&5&5>3@o+maS(fgj1a#VCO)gyglml#G(Z zz3Z3^MUZvWq6RcuM+l-6;MU=P%h4jWXXg;HCXfnN@wFjT^rz?JrR^LlH91xX3_8B#d*c&bXMXJaU=tY6HdGq&@< z^>KUe35U2jNnbZFT}5g|H)x4GWe$z8u$%L77x?Q~{IH4LIoWS_R=coLJ{{$fE?H@x zdKUQln)?zxh6dJd3xxa{dM&%xM2za=8WLr(T%cxz1^Vn;o|X6Wf@lGJ4<35VUtY{% zKiNmeL`d2j%wM60QYW4^EsRu2;@Q;#e7x?(HRR%4Dh_T*PBPw}&#g|c$rIxnu8M=k zwoKk%g-&=SH+iqOEY{yqh2pposSwL9`mO=F$aflRaB6yMfP`u_eYaGTj)}XvSQJM1 zv!Y@5sp*8>reV2m^*b`>d`f6*Pw7-Tg!$L-s#@FTIs9nRT`xOXgyOcCa&m1MKiqXv z*oLDMSEIWZ$dmJc&T<0jiJ+xdVnttL@2v2UV|)2t@TKl5uFLVw@v|c7A0E`+=7!&S z`Jg(_rN}c>q}2RsgDI8Y&ByGnqrz1uIE>)$I`hQ~=*q2&k1b zP{oP{Oro5(ngQh1MCq%!ng-EFD>fxTd`Y-i5Uf*Wz*3a3r@y!e|6`#AJ)H)t@=+D( zb=eF+F^)Tv2Cx{%naWg>wVn)!XRCo|+Wwt1h>ukR(X^G7FSq|wP6f0sw|IMTN(~oV zX;WH0pNv?E<*d?9!I&`>PNmE$eG6HMI{fWo5M|7SQ{9Q$V?h|t+j9%Tl(7;%Fi92c z3DY+hqf;@n!0A+6|IQa(zhZWQ!?jwFrwVd;#wa+soCOvIISPHDB663p0-nAqH%6XU zA#b^Jnor*Bj>B2R%$K`!kYpnCP1%G_2@>~oizz+P{X|X@lmgH! z0Du%>@3R6&;1v%dtQ3Gj5m_7Ez3^TwFlX`JE{os=#Hj*6oD!?=P5ebK;-4-6gFGc~ zKi}iC-e(F-c_!Zv@tN{$fho@>l(w;$&yYV}G_kMd z$8zfG*{ni|dCj_9(6;%3Ak>%_2vU!Ex*~oc2rcFXf>dLkAbgeALI|bi1;VOtD6DL} z_)$WrH7_tuO@$wXR`UYk^i)6)TFwguc@)9Zwqqj%^%S6T03dUS-QNAea^gZ6%HOmw$f$01GOiEqn}!_W()r`fTO0#)ZPked$bI-l@G+SMt1OpA6B_@!td$F zkQFGX8q?3$*;TaVoP;tQ9*c8Et5iv!X_$I3D|A&_C2M2blofoBevV)HlEBZPsLKLT zT2=c(SzK994wV4Dy{7s0Hd+dW{fM$|wr9D@c1ebb1+|_9*0Gt|R+{XT<}R=5r(dpV zZ7dC7DIW++haZ(>rM%MV+OXl*`cUdmdj)xa-v(tkI6dsZa963U1tD`$Dt^#y- zV;%T>0QZ`wQdveH0x92_Nn4_2)myC_*W04j86rZhmn4&}+P@U; zAJL1AIxTuEcIo+Edo#X1=ymWI-+twdxiR|Y;{ogFK+wD~>j zwR5J_&~VS+yv_5kOn-j7+tjfDt2zLSVkz%iyGNJA+T1aEC#Yg;g2vl2GJHSSz6r0o z;5enju^`~cbWm63_pI{xFx;&(V2bIz9DFZ;0{AE_Y{grH;jP;tXI9Mn%qSrM_pO9l4e#kY9NnoY1&br-;n1B{fAX=JD-1<-}N zG~P?em@-x1R~KXzIQSB{j|UCBHEy~gwnh+Z9AR#I4IhAo^ysOM)(z7)XS7U>z4q@# zIGP$>ZAonfHk-!b?7$1cDb)0Z7 zP4sK$U5AZ};oH)`1yOeV^lIDUDGx4yFM@kB6KLzL$c42#jv;6S`0W4=rHA%d@s-E| zP-LfcC)*Vm=LyjBnaHe{FH(dR>HJPvp6X~>nP#a(A*PRlgC`07P09rx?JUYP| zZWmDTdeiOq!rdG;Jc+&ZLc4!MaJr2Vv$~mlrTv}69a2JQ>VOf7 zutQ`TSvjG`*Sf$)S$wRPg#<3@bx0V+93kADsbdH*xuB9_6u+_)_c!BKFYaH%Yg6V# zvzEwiQRR6O&)B4+arrv+^Y+kR(ll!%5kw0bcflOCX=us&cO6uR>u^k2f9+~TB08Ij zirK@i)j4>6KhK*4Ye7n6?K1`NVOVs}gn`JO3*>TT)G4O9Qi`YAT;eKQU70z?v;Y;r zKdMQ$kgSJB!qi=~s@h^JnM3?2mt?t5&@J!e9$0>Wk7xwbg|I>9Lf9F+jR~(V>vgwB z-BD|+J-UG{rRj(L__Y@4$d-JFj2HJi$$N`gg0BqVoWcv2$W_}fT)un(r+GOZ4zS1q zZ$Q1czHv$aBFI6{e=IU;en_+s>Wu1uC96HG~C$|xKZ(Q zOW~ro!~h;%d3OmQ6w4t%DQ(}dInHCHK~U^qHAaa_uSHamfIgPEg^j*%%8;p|i0UN0)o0Hvs}5I_)gXvS(&(5iomnp#%#j z$`T{vq%&%Hb%}MEk5`1lQ(;a5U6*z=s#Cxde`_W1qnw`Sx)e~F)t9?@hb(ttX)F0tHTZ%GKqz*yqK0Tjy;L%P)@-pcxj8cfpRASAVjK=`F{>~`sa4R(AXecv zTg??!jVeoGO{SPkD=C7lC=)BYqedyZg7mwjMpuM?Us;XzWLlWSUo^HRHkk$4j?7Jh zuz%7dfS#6`VQCW9E8Q9eCKD-r%*lr&G3M-m4tgSp-ui%zI?3e^GU2Qst)=BCXvx%$ z^ai}F;iTy!^XjCcY}eR*xdBB0J=g2u4Fc$xtLYGikNOD(K?_aphWQ(Dyxl@?7Vot> zF=GuHuo;rrr=k;%M)7d0>)0<8DW>!1-!r!_oH@US4l}kWk2{Mal4@JLIv6gpf+;Nr zqbL5>PA+ywnzi3&1qu)GDmM0<0#o8WYQ;$eszZxccE*dN_G=8PHP5+l8{?tuWPHv} zd)!WP)VoiJ?L63|cQq+;{8*r+=r|qNuW&E>9f|E7&Z1Es0X6oef|mD{rb%A$&h=m; zoP)j1*?skW^#(HeXlHAyJ;X!(s2b|BR)aa?*}WF}g}j-Ex=Lp>E=~!)_<;v_`!y?p!RVa?@h(GBG7hdu~~f zCdx+#Ps`^L_)g4Go0uK_6&JJHIihYA1lP6*Ui_mm`&8MF&y_H{nf(fE!nl}S%qE$= zGHy<0Z{{-lii6oXuntBR=(w0&JYzi>OcLw2XQKw*ulXiP>QD`VN@o{79l$+`3P?uR z0vpXq*ECgA7N8K_lzg24=bP>k@>j|05GEA3C7^ZU;(|# z{ zZe|B~S|xq%s3`4@AlG@*4qk&dDc~ujc^_86VC@Y{WKF^8{3tBDC)QX~_d6AmK9n_AYHCh^Ht5=N z;-9`%Ub6QFcp$u=Ub3fPDobIleok_pgcv;ZvASD3TT3^Evy+YIQd-Pbz1PtqC_Zku zkH64RmXm7PRs2+-X%!ym>$7zG1bUo>z0gRg87o{FHNfK#*oY+Iz+DzKPsRETbj6KY ztyV;V)Po|wd)swJe_0xJXm$x-gFU|CTLIRz=Ds9GbuXy88fU86FDVbxb`$h~B z`V)~wfNqj9+_XD13+P0`Ki1Rf3KGOj(k zlZ(cr&{EBtFxI>j53gZdN`GmwvBJ5L8r-nYwd<9{tTg+~4qz-~M0zjGuFarOV}3>D ze+3F6)C`X71C;kr<*Iw621tbeq@{fs_7~&P zD4Zof4NT48eLzuU$(Pu@tu~V+CZ#PoTmuOGVb3-%PgX^jT{b`HWo_4le`PS(Mjx>g&RyiMA{SiNvAR(z&e!D$M6+3tB) zedb*)HSgHq$+tV5rCU7d5=Ck)786cE4=ZovC2C-xosnjv;;dL?+36SrNK9z39~4%} zj5de>^NQItg_6j60XK{=qjos+QFJ214w+||vtqEC0&Wa98otAzO;A#`-Z%LjfnN&1 z8&=^Jsn$b53t>16`|+GC=2dW8r$459X%mb>?OU=nv6e^SzXwo&Y67Lna~hUA9J z6`qunNV*(k{o1E-TNCqpte*OS`p(UJYTlR0n)+StYRhQXC!r1hTF=wlMiq;qW($-` z072z~0Y_mfd!h6*0mLMaI2`nPm~qd@>&0##H<(` z{AitA>W#)*`~yQ-jZ?GcX;7=8`<)tmXxdmP^*bI))y12u(e99(Fd9n>cv@z?VnaBg z@b&fSQTI}Nj9F0hnD(+)(bU&5x;iaR5x8*Og2qH{{{T?_T*CiS_G zQ3afQ`3n8AT#p~G!7kw-G}#l1UMQvmZp6cW+*=~0DMIf}+qyiIA@V`kzc!DRH7g;Z zEquHV?P>AA`EGlJ?i@1oKK^og94+`Qx;AxeI7|EUuV7X@|G1wXWWjSfTkUbUH=Mov z%u5#l8$Rq7)Pjl6muBKCxTuVUr|Td-km1~U*tfQaa;=me`0)Vf*55Qt_66elBJ;9i z8wHK^tm?ztW7?k&pgS#w9c}l>a7S_uY;iO-i4Z`qCX6vP(cscoB3=8J0eo;;dS|2Ezuszw zv*h>mm_uluQ{)m;*d)o99#Sc0>DZHlxdrT~@BIZU!oP1#b#Hr?S@OOW;B zwdt#@Q;Eu&ntBvsTSjgF#k7^y3RhZFV=l_TC02jh3hUG^DXdddR#;`Y>N{)N@esBZ zpr^+0wACoadTx^7lF$vK?r?ev_c`)f)AgPoi%U62(M)PFjES0VVEDu!=xcEUM zay9A}n2D;lA+2~U6PX?WSewJN;v)W^hd&75?MjBpjxWy(CJ93IkcRzhjFF;dkvY;! zY);1cs+M{9(*R1$FBVnZs5LEWl4jeLP1pIv2h67R;A1sIecw7BoZQ4DCi`uu<5Ra7 z4S#0eW2+Js;+6!~$7M;-DOd@Y&4pGb@^ZfOU+u508Mb3p6vV=+?VwV%>Xo5&utIj# z3h$BnDmeJVnxUqM(SmGV`7wm6$Yy%ZBXb{QLVjEF&sFGzS1A8XE2t~xxj2bTivOte z8N^DHItP(gr$=OJeiI?xO(L_YJvn(1K3{{?j15uBU|t&HaVqub9sK~ygu?Vgcm*1YS~ca%U9p=Eqgx&$^2E-5uo2!0cW8w)yfcsV5a&bSJ6<>gJPTv zM;lei;Hry%)o>1j=z(Rwym;vvkR5YPA8KbB`aAe$Svx(-*qB2sB-*~4byd5VgMC>$ z-4ha|CC`KL^L&%%4KpUta9UQV8`iGfp!OoX|iT|vQj!lfMV@3 zgI9m%9a&mQLt-fFrG_KH+`GPT(PREq@A;d)uGgZZ3%q1FoI|eH4rj?vlbWPu=lT5f zeuI0#m; zHwExk7vP583^_+z)LFAeA=RhJ#w&hcX}XoYxYopRPcQ@P>W8LMb9w2`<8Xp*eRn#`E<6+gom`v*iLw(KUbQ9@|_Of3JY>nC8)sdxb0AJ zQi^~8mtf!e;Ock_+X9Lg5rIR{z2L?2>p%|viV>C9RR-OF5F2pQ3m`AyeRz6f`K%f^ z=2yOm?Rv7Ky40not70BkX=r9_@?ucWDRqWzD|9lTZXzu!=#OQe4R<#RMPKG}3M zkUOUJf4T|}XjIeN!wP#){bJy6$Me{hiS#zM+EwSz_u8BB^+B(LT@+76E=UD`*L`ET zfRce!qSj8*%i0cW;9Y2sFqR`fBOBagfc4=fi6rXe&d`nF4$+s`b^gQ*`P&ntG5ecm z3NU~n=s+ml9`u!GJfc5LU@c6AWAGJlB!f6g4MSa@fq%k5muuj*O^ajy=vO%Z;9;#l8sx(-h9Kjv$Hww4*D&m0$$QE5-XB*Dm>$v9jN2L zA^8yzs3Y?{zD7s0y`;!qzmG@aa}Mg4<7*-L5s|r*7520x$M9zQJ!At4v+S1;Y&w!j z(fdO#f%{^#=0kps7*&03vX{nX(c5YH5Z729;n5T+I z*$t_r&tB%*k@%lnqj4uXN+wEHwUGS0pISDi(-2;IqlV~lY?RI-T8)`aIrzl%+Q$Rf zFBKc{FX+{>#8}C>*wrA~M@+pHp-w{6j4s5jnz>A60JAUH5i0!4A0WHH*ler>grXlRbnO zt@K`IROd$-*pQJ??Fnt>jff!qN)gf*Q+o)uB>f5_ymK6QCJ6;RjUl z%sTLpWs-DwVVY$|%Qn~@;ST`z5;LJPnPB5T<6Sbw@nz`oz^J{o-NS3HuC&LS*O$}? zycxM*i*nW=$|IlTsforZF)SJpM$0)#?zjqXv`#Zg{%8TVa5L}+qN8z0*5d^=60>g+ zV_E)vmKl6_WF>OL7v-)-luyWMa0*}tYits(l6I5kzqQMUswcv5C;6yp8YSIsme9zy zERh}TgixqZS<8_E?#2<-)L=kyuMRiq^N5M48$!8eS9sh==YkpbPt!19q`E<-vV5c2O1rAUcB>0nmSi(1;*^hhTC^mUP2^WF zAmYWO%1$$F4@<~n9lk9{8+tOS+#6DNw2bN^vSMycp&0hBkB}9UTUG+P0B)MtWfZeL zn^rIGUmIUny8zmKhEs*S!8$UkL6Ec%H|2w9WRn<>)HL4j)EeK~#(X^H=N$`#{r}#Q zWh3oU1fLsxQog=@!T@{izSV0&l0CRUD+$a60;$MuB2&tZEyrk>zcF1@rc(fZwU+jn z$y#Z2P2J98Tl&aveIj3~`aWpCZO3@A4)$5=ZjZ20;TG;{V%LnBMy1T$y{OqxhuR9r#nw-jhea`C0;1AvfTzUYQxy`$ zU{8dlqN?i@O@_D$UwJqSHO=EjQLg|4J@BR1R#>XGd@|keO|>eVPl4e{-^#2~Wn~nq z_@@4ns_#ZNr~GvR{EhOgpQvLJzhf-|H}|C_@xKP})MUUZrjjQ}1DDP-Dr~?DlFyY# znkwZyPpg1ho)||oRgSQUZXTz=#5kgPJ2-;U;P!BYbcD^$ZUK%!r{U@H{K*V8>W7IG zd;V?>&R2&;G1ik&H_fQB1UI8e5xIiNM&Yjj1vO5%(wd)GWjEnlYgIHO2Zx*O39wC- z71C&PXPNGmJ4;co+zv%eX>fZKwY*xrvZDS-dIx~So;3MH`~y~>V8|w>5@U(_@J~Vo z@DWqu3lA`C-Uw%($Dir#vrJf=Op9CY)K-uT1P?j_*V2$Hv$8gUmaT~DmDCo4LKjo+ zl!EP=QNjvs)6|;&Gp}kYMeImck^2EtJ1PYHb3H6U!yhL2S;dZlCKNDR^^zL9Woo*{&_fi-SEne9fFOJ%;F&Ni8r~H8c9?r5J zpOgQhZK_X-(CD^N=QU4ArQbLy8vSu#%A;yD3iq$;b}_mB zY#fAOBbzhE8%GIgIQ2Cnn@kx zwL^i&d91kMiZk2b+?uefD0&u(9Y{`D43C`xbQhVGn9P~O8N7D#ku2r3ti6~rQ3qxn zt?vi8>_L%9rF_{5O3G_IHYk1%Py6Pgn%bhUGePdAf*@PuPhbB>?OPSN2a(;FFz#RU zhee68D0#K9<9RrdVgipd2_vczhjnV0#qx_Rh{KA2T|$s}6%n5!W1zG&BA1Y(!$ zm@~SYCSyyT??iwq^71@}9)_L9O~c(-HcA@KcfI}T2iTGebnbXIUsIv`D2437JyZXZt%bo(YW6}Q;V9ZhZVZN>NY`YH$RIXRc= zxa0Zt`la^G(zxqarWMECSnW9OZs7rKDtfuHZ&MoZ53qY@RwpQ}QNn}D8|>wmtAdvs z?}Ec)>}ot1w%WsC`&O&pMnF%5hOn=53;)*LB)0=DVS&^()<~UHOAdm@gPDH10CQ`! z(r<01;n#X4gacxt5smNgL^PV!%^u!a*lJ(JYa?6a<}o}e9W;)?LKj;ykI7BrEo=z3 zg|AyQ(J>-_haWbU5h)}>lTe&un$oI{@)0X~CQ{^}+<2#HIo}!@s-Ogw$#Qo4bT#3V zcvGp9!QMx%QQf?upB$Tf?D~lS&U(S21(&*Ot}K(YFbBzR3ixK=s{&Xn3q;~;m`5)V z-&O)c8k&mK2mmj}Y8IzP%AYU5Tv2HTh$^gFh40EkXL{8gk#1v5V2l8afkNrv8cSkN zdaapP#eB*#TLSumb81EO*|;PnC$>5NUthgN5fydyg=rI&kbtPjEUYr$=ir*^pTSSp z;G)+$E2eK9`;DM6ReSmJkFW#b(d>ihO&DSx?XfOn+b1- zmkFLnW%KJd!|@O!%FVU+WZ{rwqdXeB?m*`F903Xa$nF-l2e@hX-8@Jn=VmWVMAytKF0g4BMj;znC3JAZlpJQQKCwIX zuuYpUYz}t%_{L6cPbKsl#vK9PSd`81B>81Z`B-3*m}FZz&p4VVH%p39R)MudNAA!2ZKl$a$Dh-RD@s7MFgD}hKO77sLCL7f52`o z4c;$dfK*)?jC~nKk7qTMZyi2OQwTB|E?%An#-69>7vU4RB)|@rj0EQ@k^q|)@5*ZX z;x3m2KI>3Em*OGm)JA`qG8788n2xTnKWp8`#k@*e#D$EB+Y7U6Gix&>X=Y_n3KP*D z3^?m^botO27>>*sz+=6o*-we?SEj1n?>2HUlG#(%8Ls=kIcOO7Xj5yE3sO)5(RmplO}bfbbRTvrvrat+{UsEE|$-r>698D zK@w9KIE97r!t)mbh_TdcJRJ0TxCN!$hQe^wue#OWVQ1oTm(-xNHnBE&iTr4tT`%?_ragYL_PPy(X~(yaq4p)OMUqYxvBzwwg$U| zgHU!)D9WOkVwk}F@+A{=4`oPv5caRl;}!GRxfUr4AFo4uTCBjY2uHr&$6rQMV{E~8 z(Y2{#ljiJuU_U+VhX)+D+IXFNID7e-mo7BCxf_162D=5d;N|nBdHG7WJ;G+vZRKvJ z%wAY91?&DmhLh)E-`XC=#h5*#yVujO>K8~WJLF+jLF;4BZ60FQ&d;imMV(}#yVGLd z(RQy(?s{Xtlde5J2xS5HtUSTWy{WAWM+lE(5zM09BV06ng_$@lSOeb2izFzrc;Ixl z@u7?yT!xE&sY1V};OD0(Oqe)mtrp6Gm>P+E>~U3Ytj9Otw`=f$X~`4Cgx!-jHGbeM zzSfMUCPjQf5{C11ju+vtg6Vk<>)mTxgKnpF8m|LnFQe%$WC{GgV~Wg$ktX6!@4%TF zb4eO|1ZzqefH!}Sv?fzuv-2!^!XI86=^AmVHm9 zCEHQ(TT_;gGrJ@oXQC-DVhPT|)}>R`ho_ifo7x;yzfDi2li|;lVA+q?rf<^DCMs)c zs$Ga}8MTc!X-t>jPjRI+HRd8N&8ZTrKW&9|c9#^^*(ocmQzaA@9v?LvU@@iKaWj4t zGf>(2>x3DE4EEqJ@d*i;n~F`%~*oh zqQ^?inC1dYD$vtso+pTx2Scp;iJW!6T^T(42J<|8PtDY4j`2IY{Sd2=@R;@H*k1Li z?t$8$VxzEyi^_0WqWzwdHFWQUwGidiQA6ioTW>`H-&eC&s;YPiPB8QWWJfr%GUp9M z*+WIL@@rC5v#Tuli<-UKi_wV|Qx@T;{KArz>G`wFqmdNcmV9>=I^h+{chib9i#bz1 zlqdC{lcoIX^e9jLgGAzYlk#k8Pt&moe^P@luvsvrbm*lao_JJG-qH`S`YKF6gr{x| z=cJfQPCsiHkZag{xmmWCmvcVJA454uGb#0A*~Ed*8=F%XB#ThchWZ(Ww1V%BHYSCX zU1yqVr|bs1R?-TP!bC%98>1dkHOqze29&Zq>oFQb%bS2)M`R$q0tU-}}8N0X0f`#h*m4)ejW?|>Me^dK-z zA#s{PJH)oSCT zrX)>)=r5U3(4IgQfc4Kw1GelRUEkBYIotK^G1!u)h_xA#4)1Ck`3FSf)bRYFxq3LW z2Cs{Mo&@>(NyiAwd|F3&hqRrN1$35yXg7Hoje5pT%mDFgp5bo?(wWdExzHYA^XM785BHn>%de|usyW`FaAkKE+EAk>8_Zc3XRqc}2N zGk%pI;v3-}lV$?GKCxmN_GCDcFrf4?PDo%?C<6}4cpv;d245nt2OIR`*7d=7Y0z&` zFF|-^mOO#6;e4vv85NO0MUH;;sy@7;FV> z7liJ3-5zb2j2o;=l<4JaGI?#Gtr|O6*9E&(9^eZ=2gSXDS9aRHNSJ1hntab^E@_po zfoRo;kPG!y9{Cde0ZyFMs0 z!|T|ub{KbdHfcxIZokvrjKxW4gYS~MH^`7L1h! zHNmXNq@XK=W)X$pslU-W3+X2c0nx!lUDyF<2U$?@A*7@v7`|lRM#DRp2&o*ZND=i|I5<%8SPvltZ*f|TMyla(cG2x zDDHLpvHY4B17Lh0mX^SX2aqnI@CY{ur;~&y%OE%R1HZw)NE&OpY{Xcz^-~D#+{{7R zm~ZMC%8|FPnV5zmBV#o_dqSEr`~xpYQqdYCmMvDPw<;&(K;=#dUj+AH#=P5Kx;AWg zx+n!B;aV_49Txb{o!^TIw_2EzE^{MJd<;Ga_oqiDbF^-aTJn1ADzV?k12~$VfOrzz zpD?2p*n^S7sCYlGeh+^Vr&q!=VYLlMd8C}Qy&u`v6Xoyl@WuTlN4%(ef%wj*y zweq>)HB`*E53W_y^AdH0!4~{K9F&acmhK?{L6~-O(##e!((@Mp7kDH9B|+^j-Kq(B<@nlq3B6fz9FeN zJcWmAGRQ2$4ZNXKN@#6|=-5okgRG6YN)B5HDZOee8?{D5;y{8WjFDs5CNagSSMuUi z&k*BR4RI7hxC{DVdOY>Wd#atd4eaQ+YY@Gmfwf>5ejD$6$REd=p5P>M zklxusbh-?IsmKhUiZ#WOSaF3;QKO(h%0Qv%_UiJV+wLN3&S~OyROT zZj@fl_S(4d@&On<=!Ko&a` zz&UZ(LNr1AENQahc#|6{-(ta!1c^Lpgf1)FJ*lwLB{zt^4NuT|0f@Aw8$4A&6g%=1 z^`)COu8?#FEKIQpltMm>Ks{_@ZRM*-`RIh-bFF34K-Gr6GA08j%dHBp`tgKaI5V>) z+C&nsaaIcFqww~yVH8d-Ysz>?5!<@JSJtf3s*GU6Uz1i}rYlE-TmfE_v^fcnhHv+$ z4RXOpVc8-veD98_`4{5R_4anWM1P7+CG)0a%3*iyd!rlowgR(<`#2a;B{drr$GC9- z_HHoTN&UJ5C&{t6ssD%%ZZs_`DyY<5RHe_IfALfR|1y9RKJ#vLdoi{`#Gg@-HHiF? zlBDq3Bq#My8z^|=E;8p}d(za`C5*R}d950sG2F*r zLsL44paFDd2AXdJ#5Kd@g@B5{{QY8!=DF z=irguCVr&AGIC!<(6n;(>u=!=x88a|^TUc#wHf(;LwIDopx#P4r+bTS`E1Fetk#*F zTZZMH{)lFLv!aU;l|WHwJu6w^Cj)CvBTG7it7h{WvtOqET48$SWAFn-bcN!%*%BLB z$OYP!-8a0g%bx%lAI_iD)RFUloDjt??I4c5z$?z&F|Y8`oD1c6fLPO@!0BXi?!%T% zp~(y8I65rvvY#)N@^;`40ytw)o65Y#;?M9N?H!&K%d$S-=qg(I`0POjW~fg3-h#X5_ou^g(UFissvI!PwV3s6Hf8qUgQ)^;$%(*72%r%N-wx@$Jzrf z%7ctoJf5alM30pe{(`{#CKn@pk`baXaT5YtBtQ4XjOeoWZ=o?qQe&gw|bmWku73NMzxia`UPQ zC&NecHm`K*_^s7v-8+3bL_sc0_T`H(uY0#on^oEI^wA9o_1-P1w+hYhWHIU?zl^Lp zhDg(j;(?g9SCk9BQa*;mF{#1#2Jo;&1*x-Q+g%YU7TDobM<834<@eI9oiUfJz%!E{(?Rq@`*VaI^b0sT7uFb)X2eT6fmTLYpcs zutEE+H6Oi3X)Br!7F!EnZnthhWtNzAgsnm?`(|%V_0HgP_FXImDpKjjTFVM0tflKL zl&>PZ22CB1E6F?*r*h!hU;HE6MwgKa)(uvf=Bu)HnX}&Tzt^m}O$ELuZuQ$+-OcrJ zcf5H$JgB|3hEK}(sR;OJcr*z<<5A7(BOJ7law4R|LGqgP*5C*2afd>UOdhq$l4ez{ zr}FxLylj|v@>Ri4XAhe%t9(HcM0lL_N&@cCQ1EB=atZ|?k+)QK)@_lY@-4=d9-4vU z>+)VFLJ9bNuQkr(O$w)4BU?&dp^nBC0=;xld@_@RI|dJ~MOqF{djtQvnGlfY?yO;@vNn+OWVUieOs$mkJ#26+S+W$Fi zhX5E+7K>O|A&D_4(iSf!FHGWN_=iae5xnuUfHo1l^TH&RMwtHK6>|lRp*jd&@5Za` z3it2H2|V~L)x#tW+^p@MOqirWtA$C^aISMu3Lqy;Qla$;lho%~$m+}O&M-+EHt*sL zEz~LMtf&Ob3zPV>*)XY!m}L3R(`U>HlN4ZiUn-ZeRmp%S-(6vnGB67kC!(LSy)kXk zS6-OJ;nQ54B4HB$etww5AIuAr_+$BD5`VB#n50fd)}?Zo#0Qj7!6Yn0X_n6klXO^< z`eBzt%)g^?i?s@y_?r48OSdM&{T0F_4v=tuJ;Nmat^6=a$9F1Wk_rp_CwG{{pUs3x zw&OvW4XNxF80x#C93mEd>LC+H*E3|&iDRvsSJ+r^&6GnXvI?%9CEBXCKpjm@E&#d5 z$Y$8mvl73SF}$n6InPiPDeI}_)z@&`jfFI2>T6V5I=8>o2i5KmW$J72en;EBkV5H8 zqQ1r>hf6EGyw<`yY|kmS5%lmWw&BkrLe63v{+MxXD;H1vtQU7@w4uj3p%SoWca`O& z@{+2S@Lh~VNXZ4vmhE3jeeCkI(X>9l_E~T zTAK8v-c*sADW~LE^@N45B1(?YN0P%7ah38sBy~k_Vp+;7yh=mHe%O0%oq#c~C8(#e%8@KLo5=py9`{QLRk3jD#mas?Gj zA5&1NTtS_T*p0k$g?e69Hy^NSxq=2)X_n6^SI}YgD_7v((YVD~uAom+pDi9 zRdw@m#KyB{xq>>oHM&-uIZ@Tk(=wHO5l78K9sbb1L8pM@oqp0|9A{kuCyB-i!uLwo zCA=$B%KbKYWbLh5^|t*D{2%VRZVriOkph2OY-+gby7{AmSBhO@ooRk0i30;?JV3)Z zEmRV@Hrzcc7btyK0B^Pgj)Y6tj7pePp!IR`{c1rH*6!P4TRDXckseGAz8a3bdZ8*- zqh6%XvxJ0gOIsokz{Xp-xagr#?&IXRs~=wF*Uo~Dq!fv_ZAoICGD%3J6&YKvOs%j6 z*)nhVd-nZj38o<&Do?nqh4u`F{2|psp-DpO={|p^manrYO+^@Fs{k;qfFEZ&{S2zY z*rMjuYimJFdCr6HcOM_45M_g(+pu(>MUkZC$F}2h!<<}t_3c*j- zoafvrY?|Ma4tQMYx`flBT&+!lo!qggEy))F5$NQ`C=;M5e4pE`q%_C`IhwAw8Iy_f zKXLYR*_FeY1URm9_gUU>y8AjICO~a@fSW!m*_5iro(8k0)oB30J)yeexxwXtlV zObwBWid74xNnN#u2ro3U=?i75OFlJ39DZ6ulv#kcWl{q7!+mS_=vjc;+_A7WH!pbs za>}a(#e9l~__K(Kvv`Ow|GQH)-tj&XP$JC+VQ(Bqhcz<$$@sA=ho%E-NM~iB4-&hc*Yv zW$O|ooVG4`tg3T_jeBi@hQto70yA)4UVfA!dJ&^0SBUhu&4mj+0#+iM`y^rzm0Ls! zQCWxRr`=uzUCE3)l#38?hWz-bkX$F&YPOG*>E+IbG)_B0T-x(9LDUT@$0#}eGn16W z3Gze3=|9sUjS@eBlV&;ov&t^2`;&_niJ6!Eb&52*8XpwmCHMyWtp-xHSR5bc1*|F! zBIkKD^+|Mku&?u3%bsBhpsZC_Y`QB&pw1XAySh%(C)4OB@>ie%iiDl3kz%WaeVtvN zpaN!=D4MI|3j0*DARAU8OhG?TpSc)mzS@2>&N|P!F!TSBcsLlfdfgjwIR7$veGy+q zA$LAd3$j0|Ntz z7GwXYEUBc8e~;{ce;QW9rmO$G342s)g#`Q)xdKj=b|ROmQazMbs)goG#LiV^#`uu} z!SIzDMx5}V3N@-=H|-lY%H#>SSy-IfmcavQBZx z>H*Q$H9UW4t{%>;!Rz9mXD!=r+6%V~kg%kgk*%yQuwj@iLZ#>!s3&k5cfH7teMe}_YIj0nQ^W=bq?+e5>~vN z-Im_agh`Uxc#_-f}xJoj1`WOlz5VpnE zfxxxkHCYdg+FRSbxYN4Q#+zi8p6|6cKw<3yeKFZNpb(t^b zf_&xfm;LAVO}+#kp)W0RVBOb~BN*x9ayn)NR<4#eKkdOD-urqIR@7zIeZAzNiIq70 zxR0PEn`gEqu-%+1AvP|-ba%_-+A>nTwGI`wTsic2%5anc4AY`>)2s@$>O;YiHaZ zcL)9OaOR^(bTn4%J0^rS+v8SK;97SXSok`7NtZIF;xjPd@)yN*5gmY%%XCF}lR zrra7}ZfzE^uC2{9PD)qwVR&z(X$se65H=2z~`_H3%R2p@$9?Ng(DR;oALldmR{;63>-*(+_7MO1YhTg5A_7&TtF z=gB_ua&$XnB|-Ku?C*|RXnXN8<^C8Gv@?zS;mCG-*zWb>UaN}`QX`JJ+l9r(lK@wT z?X9?l_p8R?nb20~kU^gkflM=wZ;ovo1z%m1pxj#N1a&auF5u?NJ+38yPZcG^s)@6l zqT!nV4k8n|>iFGh5zp#aCS>cckqdvtvA!#<5q%%-xfBnt#nyf746`zSM5So-WGN6@ z%S)G$-=7sil+avc#wnm3`bSTdLn}?BM2}5jk!@kczpcS@6D6vazOj`}WLQ&MFSg!m%YUs2 z`=f=G3O5l9`_f898k!MPO1q>~Qqy8b+`C$Xo$>b0xHTeq_)&NQ9yS@GtZjngYKPCB zmZ*|yM_UN^qrz%OgC<*%RbK6AfEAVw4J=c7cmisN&mnD#uip;;p|Bp((8*G{iXPJF zTtW|NpedC0rN%@pL25~$xMJcSTSdhr{l48*OsQ#iSuye1C#0CPg(#tzG-$FSQ(iG? zfE89u8d!?B$XbvIC?-C~ZYU-Voh+5BC?<{0B@~ke+MPz}e{8Q%%lL~l$tNNil3566 z$|ii5wJ1g7Z;%Bs4}eAR$HH*JQ%4 z*J0=z$S7MHFAXsntL}Wz4=Qcg-Q0aJmRQnz(ve)w=HXK^7x7M53(s0M4_QEevD=T^ z!fn}XymY?X9-ZrS+*^vDsKad|sMt(!Z@~&dI@*{NK^%JpnVp5ev7uj^ z(AEO{U>*8X=W5&L?xeXo8hPhyCZ{L_lLQlE5}Qf-*eqO`=32cFXDSugN6~H2zdjq$ zOaYZyN`}`GEmC-Xu>E2)+GQzFubJ|`JaL{Jgugz07FfDK_BX}p91A-(PDvwyo&UqP zI*r7$Xcxxa@hz+s^giqZzgpWBWUsc`o15W2$_vN{yGGx#1jJhiaPxWd2HRAwg)>e& zf#-?vMD12j$SvJoL@?)`V2rTYfB0Jd6Qu_YU|EH8k6zPi7pkADB({{OdaIRR`iU>s~P zkPIZu$#ybXCXi*AWWr>fgyft+l1WI0WkMzkhXY9_%bAm8GAB7Rx3>4H-m9utZ>#ro z#^xZY``53!b?>cPcdr$w7SSLGR}ji4{&A}Wv(mwKfB3VV`I3@z3^IW8_vPr9uCz&? zEKCA$f}I<#`a9wNo{mR@a6w# zgE{<1XV2VVGF_*?IMSUSfnqUM`N)2n7Zt6Kx&sC$Br|`6vH+t-u=de(nN+a8$1S!j z!GcywPq_mOc$H+jQAP6R})ymPRbykiNLd){Gy)yq2(5Hjz)C7}%I>=#+}p{{MF(vZF| zD)r_z>)BaHAog%O{YgA(cjlZaswF5i8&l!bv)Pp06xL=VvkP}5@0RZO(!}oXw#K`Z z+eT+&dUa>5x6vB{$EjB^{hBAAxb=+eTNT?isV2 zp#$_R>iefNW$YFp^bfSr*^17BihBi8mmaTiL06rdBn(8Ckb}ZxCC@>@Z`#~wOCtFi zh{UZ}i*-lQ$%LE0meGYY0_e$oOJN8Oi!p^P## z1rHYS%ZhUF0R3PY7?HgF#qmLX`Ptlg<-71B_Z@UHLd_%9K-*+x<-?VIPNjjZSsj-NwD_9{R-=Dj!PjLod!Eg~DUiyUNiYS~aMk zHidj;(x1ZKy|?CLm8AWZQQJw$-UBnBuu72Q>V{`zmJ1<7y0#C58t-e+tNd3QL;kmp zdtE5xB%Pv5xVYcnD!7r@rCkD6W^aSLa;`w{whQCvvZbumm(xnjs`tMFb{1o?4; zqT32Fy!M(dh9`POzg*{P5>Jcyl18da`elittQVk5mM)g6gVEo4s4U?qUC!32*($j$ zSSH4c*=C3FOBtb0XDIQqf{tIv#h4-W&{TinhRUt#5JBze8;TqvsD$9H^vmM7;%-9U7`AOGdk=a%H;4do$9qTL8j&I} zz|%RrcZpz05$|1MGU1Mln9&wh_(kfhGA9XvWCrB&-t8^H2*}~ROKv4*2$Y1IEE;@H z@7>;Vymv{ANajFsd{FJit=?})?#1t{WcA)9aild1isQ@Hd$-qY7&!Od>5zn-Y?6VH z)h}k6*k~l!%EY*#kjs0Q4kb4hin9nkQtZ;pn%rK!F_Fi6$IwUhqVmm!jLLpW*=i2& zU4lJwT-`bBFbS9 z*@OC+3~I(7XE&wE1Ulvyl#Bw$Ke#y;O5*s9 zNP8r7^d3fs{2TH(dIh{$i$nD1*}dsT^h(M_RxM800jL%wq1e(wk8Xz95|lk@C+KfI zblldLuBrfF&23mo2<>!pE4&WjU&KDaKLlB$A_PX=F)=>jCgAD{bkSOO+U=Ygjn}(V ze=8U#x7~XfLGHx;rR_L=YM23ZkM$AEg45tG>uFhlB?XV@6K(rSf>cO#^b*8c$$Stz zV2uz<=v7*O*=NpP0{&DRt(MHHDK=LnvY=8j=_}3!QE)oE!rqGB?BAd13QOf2LC~SE zWoRW%8;!E+@OZ0t|6-~6f8esnA-eiCEuyR zh{=Hk2vJ=l;?}Ue^wUmbSqgU%T<^W|%9j^8!>mO=1dLcZ@IJTtpD7 zW)|Yu4wznMG0dLcYzjW*uVAi>oIpZ-g@Y6k7975}D0zuhJWJ;F9=* zC4}U`aZ^54Vj=qS7a(ZQ7P5CwoJ5u|`n4*MEk*y!M|GjlW%)@2=wDmtEu|l0ZFWHU z_@vnaZ>>}=Xc7$+UAnCAd!Dd`KG8x|UMy1LK_Sg5Y?U}Q2E?R zPkXk?y^?e#y;;=as^#VA$rie=tkbXYgxI?*>@>3AO@&N;6&s_UY@=^47Y$4vsRPK? zP7q6s++P|-;xy!(;>~tLW$a#bkH1pbfKJlE)+@3bcWFLZHMt~Tap^`y;DThUA<6DX z_xkr#Kr(Q|rkD`u2g`9U)9ZygnNN}qwOgptdKKF8aCflT^DkZAxy|$Od(YcvXD)#M z<1BPn>o5v| zokGg6VqVFKhbeXxp2BSz9vs*Xua$?|;lFz5o+6S?0Z{C9qiXQDVD-l1?u0vMC)z=t zY~N1*#8u?SO5Q&-+V$J;M{sY+6H~;(V>K>3_92THtoH5b>s~SAPQgABKgWI470}9V zqhIlgxm^?+CAUmxk-fK5%_%eyFWfl#(-yii&a1${M>`At4E{T?R!>+^P)c)4YN{Xw zE7yj@-4%Q2Rh?27N?BWT9G8$M9$%@{r9d?q(!AB*LjDT6RV}#EKfONcuYrG4e`SX@ zp*On|ZG#6bqYn8{LjG8s{i*_$xM2m{U=505=fSF$Jl-9i>FkU6F}8gZ&d@V6=qaxT z+$j4b`&TG`9ekuQ5Q+hh?`|SgV9zkst3A7rP)>HEm&SSs#D2u_r)Qh9uft-Cu@88m zHTZ;w-s!(xX%1F($l~X^ghqeSqKz7)=sj|Ev_Xflph&Ggw+wx*g&wZKB@T}(`EB?k(1gl@C+av&0FdDH<`-dx4w;{6 zq3^5*oARsI4~5?2)2QdK5)Hj7XEBS&Sig*|vn&wkX>}owoYP1vuL}%nhQ*%1^=50J zW+s&zJyRd$$nC}@e_eDV>|^RoQoe4?g_BB4N1tn>chxID6;FECMI5UCR2-1jsJsf7 zt8J14C$NXx=QiLrlyu$q)yrL}PF8Cq$7L-}R&LDZnjzL${6HmWOd`*L?)t0kT6EWy z&2_&Ge*`rt&hUD54LYVWTO|}yv%3=c7hBQHQfrqti_pyR{@SB{Yfl^d=Fawd8=bZO z`UKafqA#>-$yPeno1E!x^!y9iKWZT*W}!w#*sM!CBJ%G*TaQze0-14!ejQOzFyFv2 zN6gB1*KtJ1`ho$eGj6lv7yU{Ly`zRhPj^Jj5*{R)hNk7)&>8wDMHIs6-9vWt{DdS-r4?R~W0VOlDKJrmRGp9yYz7dMRCH!RFl2&1A)<+;}Y*wEe zUZuSWISP%qJ|&Cu{dKchg*N9W+o(a?<5p*Vhp3R0b6|J=X1f;k7VHJA-eR+%Htjli zx{h_4(y`{MO9q1Y(BvXB`A3{1h}aw5~BNtxYHNRJ)#AEZ&oWbr}7Q z2Wv3jxOe02H7*W)m;c8M`mQ>}hOVKY(BQO=EE!IfC%rnz66enb<@+(jSfBAwMxd|H zpl6!IVIbPK-M-cvPW#hyK^=jw@QMiUNgzZvJlq;pe zZYd5rd8IX}3Ks>L0`dOpOIHE8sP;E8f7{~6tzvU!8MG|G+>&GO38}=!WII5L_A{3SIo?z^`c04!LnXJ##cRNh3=&r4G z_6ZGsczd5p8#wQe5T2#G&%bfl_$ZDSY!uPPN0nmcKt9Yk?RzAkM(MV%zwc2RgklDd zJMVY~sG#OKN4q1=J8idpk5cy~Wf#@k=SF~)oaXbb?~wq%^6bsU@{pc&<}*!&jF!G+ zsYk52Cz7xu$}5TVCgim5kpPdH_EHK{gD6j<{a2yOHoD{PV9*RvRNJ@ut*A*xOgw&CyNKISWJZWkW7ti%ve(5GbNtv&^WGBBye z+HL_V<%!RkRL`;QIj!v$Hp9~UqTKqL=;1gzbd6vC;9V#jl5{?XWaOz(CpoiXI*D@*W|lr76PJ;;H}6OH%|` z7^4O5N>h@HyDm);AkgKy2xSA#N>c=|I8_`L4G62p`7^q!n&Y8k7_z@FotCB~khwY< zPRFBx@USshZ#9&r2!MrGjLXs#Vf_S`*=Alv>{xd(i?Evzo6aA_+powAtGw|+&fGA_ zo^~cq*P#oMh#NR?EH~_0U8N8&73(Y-z0k~&lzj8gX@ZxqKc%ize}Y#Ugn~KbP{+&w zRitB{ClA{(epm8?+?lxE1TTRG#AkcHC3p$&t8qC zH4AOKP4G&^a7_tbsc;p_O{gHc<}|@86+1$7O^&TK=u5rpN%)SN$TXGsIPLmp9U&=I6UwBDeZAqxUIP4G%(S(eko6TB`$ zTh^xMpx-a<thUQyPTk+7i4{8Gtb-_ur}V0Suz8g8Bou30|rA5meCn61-BObe`ZPKx9tv z%3EpRG{H;Qjar09YF=##UYTGlPZ@QZ;FV*KTpewYurDZ5tJh=E(~chpVFtz#{;-|9 zwjDw=NEcQP?yzkKw9*^&R;L}-rr@#Ocr@t@`e%EB&z#?*8Q#Tn{Zb1fMwo5qECozB z-M|SBv-yODmEt`j+IjL)3nL+mZ${}zb&wLbTz30Xi>155It?Krzm)ceKI^!ZbtEN0 z$*AF_7IA$+q&aM|WFf61V%er9IDa2T!Xq)Txv!;;w?+1s5XW=vCL`^hAaQlFY1|p+N!WO?t$>#ih7~}T1H=@2nR7lD>a6PzwABK^O zyysEF`!FK6IDQ|7Ev>Y8&$A*N!GI5QdLygXSrLu^2H~fsCrF+jMK}Uh6%IWcF6P{`AZA`c|;Wa2+&M;){{c`_4 z47-?g90ty>8j5fPDu-8$%OV_M{kRmIt-GW~fGxz0HGhR?#CZ|UfjWwCa<(D3&9>d7AY$qkrt+V8;F&T4Nkn8SZ`_RI|? z({=g_(tCby$O^MrdRIFy;9}N5Y#2Xe3L>~K;4#u(3L-R!^3>l}ekBgSD>DWv z+;(WTl=_NS=o>^q8g_8%L=vU6IU3PcMI#KYn%#D2wxoX|AmO%v$0%2#%uZu~!hrI4 zOu})8<_t8bvw+7)gr)~AoEPvIu|<>(Y_w6WqvrlE0{_|6<}|QFv#J*fw!}lZ)3wIf9<4k8(uK;!1p7bfex5L;HWX-emMSm+#BRj*_6A%er}8 z#G&?G#er~*&sR|^Z=D#;`(L8JY@r6+hLVo`zWQ%xqdsko2p1JgdW$m7oz_`LeB`*!+C)mx@Znca-X|Y^ z%W<7`bT+^jR70J$xW0hLpkYRJ4ukerTF7bTlY~jhq^+Y$_dp2=SFS6qB@D*w+L}q3 zj0u^n%6Jsso2`Wf&~xn?kAP&})<-^S;IeBWV**nZ9p#)?T1%%&YvR^NAZp}RUTJ+m zW9NnnLji*qH&uOiO?D-OE~0pnWa+xCv{tYgj;vH<>pHKrHg1nwyY(HSLQ<}JGa0$A zv}WY0tI}HJ7RQy=Y-wfW87i$2nmOKIdz3U-1`pENUJt4V`|A_Gb=?I#r03~vWAx>E z{N-V(S7i6rk|gW+xkJJ3R}xeZ{ZS>$`BQ1lcr{*QSWBffgM~4g<4S9J@j5E4B~q|Z zs_+<{PzCF>(ppB0>l#=IDGjZ(mN8`QCvjhCEg>_>Vc`C%snS};AS7@&uIHB!qvEnA zlw8M=WY1_{av9NG>7QO7_1D68{pgRkj=O6R(T#ox+*)g5;~nA3cG%1kD-tQK4y-8} zs)@LDv-*wiz5dLhwY8(c(R|&Eb=?>odZjH8CTd)Fg$YSFqr?6ol8}^d@3gNo#_>_V zwX0=K6;#VPV`4!#T6LZ?2F(#*cF`;1_?$`dJbBTD$H$(u@CcK$6cpMWfp)9F|0c3? z8~;}mk{caMffRCr6tGU&;J!wu-A8cJ>Ry-KMNHU|+_SJPgKdpC&kg+fH-XMwwI4w6bWZD71X6r_=IXK@b!XXKnI zbfQCRYh1}|c5@J6FX~gPiJ`k|4w3;vV@$od%|SBnkcZV9bOfmo%~sxCeR!Rp!OPL_ zIIP7K=o#jfzss48qxbYWOfnZxpcE^dFouY_F(Qw?&_;EbKlLS<3m65aa+qW;EUYA@ zV39^tmEP#G_5-2ccbH@@>`TaoX~a2w+L^Erkf0=P2~QCGq(dLAu$?*+V+1e=Yq@I} zBCdRWNi9}WGBr5^)LRz;+E+J+p_H@Bkih=a&t z(xo@nXMsN&YlKBsyHkG~IGo|{6AOWRl{1-~5Y!q*cbJDDu&Yi8)j*JgEDS9qV!3M% zq2Nv$6O%0PP+f$7-~^nR+3V2G&S-PGu{rHbu&z1ruSAzG`Tj-Q{xAQ}^V{1V_s~~7 zhuto;+c8J5t=cR3Jj}g)`9cVDU$EqP=<_Y)?gTIQyOTqZx3EANhW$v>_=F#VIz`*; z73iY1?zG!EH5#vjq}vK<>$lx|7-E@me%X#LT-ridBoyH2qb{3u#LyYPeObQ!yS)cp z-4ro^M3R1|GawWv2>sOz`k*tfL<|c$K`q!Lb@Ds7Rh^MWi5Znq@|+hYUcT1@i{5|S zQ|4YeRCzL77=Js1o@*MnNw!H7Bz}|l?qd#LLJ^y`f*)4W11i5K>?fip7FXfT$ z6ldbHJDmRHZ>4_+jgztt)nOH4XiPWM4{$pvxQvIBjnSmn>5}S9OPg;|{_aWmu802K zrS_)ZACKtXBs}Pr?eVs=z45R&m>c)HVdepOBt1jB@si@PS10tmi_t|(+x^zIMQRTG zqr>hSDwKmH=cmnGD*!bhLh!74Ctp4<_^f%eX~dPkgU?u0J&iuwLQguo)-?zRx65S8 zd+JU?C8>{2`@Krr0Ri)LSbdsUVsmz(A+OQP#c9_em$)_d!t@i}tw*eZtT z|FlqpGy$nukvn_9$arP7ELu#;zt6NCx|S?A$@THt4_6^kSdv0-$ecwVY@_$qBpNl| z+P6D%#;JHIcxSE4L)OS&&NzGxSVn<`qt;oRQHG5E6ui%!r&eA+&a1O^hwWwX=uftJ zBa5Lo27O}&xqA{whFQmKlArb+^=fSmVz@AV`_af%`+g6ExBqUpu3DLj?5Fuu#{&8T zuZ}n?bwAQ*>&Z@cb(Omg0v-x}!SW^BJfDRzXD)#M#L{Qa=zm6^a%ua8PV&U2;LNZ$ z6Ke=;B%le!3w9>S{>uwY&n1?m?wI1#xbDE6PG}(5g1_ZRZ+yB3*^sV8oB^77hrJw? zU`SbFvoA?_pf5t(S*F2K>Np^Uxw?z~~j z!y=vbc#3HKj25EGh*0zkE~6nzJQ9Vbv*^#<{Ahpbu=Se~7KD-egWlfq3Lh zf6$vd-WyIv<3rbSO%4sp(~0JK6ammn~wI=!k{+-MI&p&>Y6XG!a3XEk>qH z;d}IbwUb1O)z?fA8lEb@2@;nO8Q&#BKkTxbppr$kar&AmMWgigcWiax_RG*ewj4gl zYM>GhYD>X(cg3(NP*I<#f(E$Rre#{gWeGuvey;7X-q&i2?)}b6u$mrb|M?k*!4DZj zhe@64U8hb>dQ*Rg_)}1yUFWV|RYnHW;GKmPRjbpUES9P^`r=Goo|2V8f3*k39sVnL z)|t&C#}{=`Vuw`?G7Y5z`>XuTLddhCD?0eWKj`-^tm#JkkBID^gtPt2;39bwdW%GA9 z%Pdo@4vA%u>*G#}%OL2acpVkMGGvrA4)e<7<>QQ+L#EeVDJ*Y@;1m6g%d|2X03EAI zX=U}^doHueRLBwP&9jU6k3{KbPkHD)p8qbl>1XnW2)lu7`?AXpC0s*IBtG%nvX}DQ zvMZJ^gWR&HbaGaaUzS#phkn{+M?dY}q7*ZCoo%$hL>fme8D^;@`@?3CEKD+U_bQy0 zWYCh$eTj~ELZzgWh=IbW*ENZ#*Cps;q3n6-0`wOyqgyIW#$5pOf_?72CBdaGtHWppdX!{R7v=obyVhQk9|DXtZqI_&6*D} zjsCOC6jv2L%E@N#qG2jn;<;w2YzV#xb?eD}IUn?7&0loZM&Rzy$_lX#ppxJ8WZTdE zre}@@qqOyGz6PWF!gFM6VDCns@X*!t8ammVcIP`Aqepta zdO-p_AD!rq``wd+-iZZVd@(uG9rxBc!|r;Iyy=%0&`Z#EV6$nrKkSY9^>S_uKt&-RC>Nt2UM?Z~^|%h66C za38uL_GoWRGOeZ)J9yQ&a1l8Lck&VZTgk2EW*ZG0A&I*U zR;N4ZOi4Z}ch(5ft37Kdm4~V%Cao`D_Mv$!$%njl+aeSUK8}k9kA;`OZRi&NCi5j= zw-fXa9(qef*f=cqQA(a#*ij{0vb!uk-9{&>*|$G)OMfs39PwCCDfR5F2mCH(9|wLr z({uRJ-qWsxU7u~C!VU27;g#+Jh0L$usNq~IS;e^=ZJ{cT*!xaq-&C{Xd)w%aQusJ_ z*_49iprRvYx^SvsyIJ*hm`a?Fx6tjS5GQtBmO`B9hOL6xQ*rZ53!NzgH(;(=U&PJ2 zd&QI9M*rlY2P?yebQKHgh}M`=7>-rl*9f{h-B;Dl*=6jE(}Id=*tiS%LoIYmImDB$ zkE*h0HBtQ)`^NLo4?dbd$ea%*rKBQ0c-m_HKDs3oRGlwHMV4r8}Qb6%Dq?9 zuEIO>*V47ykkqCdv(D280x?Y!5%wURztpoJ-`FOUD%9hgatHnUbWK*+Q2b`hfQwX3 zR$L|#sLjop<(|UdNY`P-ZDiD8g|Ft`uZeBKllWGi|4N+igI;$$gnZS`7;;Sc6(`*m znhZHb?04k1m(V@vQkq^%rYDK0eA;AomwlRXa#|=VBLQm#Ac<5@ZT9lL$QPjPZA|)u z(a{(Ej}PkboGu2*n{&TfG{16>28>N?WqwfNs!qCC;Cck`m|7h2VMqgc{ZE_}+1MV*@I` z(s{*CC(KlP@#aNB&3`gVjxzmHuNvK69|GsnKPI!L9-%9^(h%K$-Ozsq${dm+( z463NV%k!Ud+fB^i2tfa{gc6=#mh{F0`tKfkk@_r7rrp)E{>%cjJd4mw_tE~uZ|x%9 z6Wr;0#!y*4gL=BnNtIz&@&Ne7%NMcwb8pF$+HaT&*gfzZ_--bT|6Owfw z(rAE=TL2ly`)j1*G)@XT+v{y~*81y!o$Dgs5;}=qqK1r2j(ZQ9S4oCApdju>DP(}Y zL$`a-t*Dx<$4+1dFFF$%GysXOQEyPV!~L0+hsM)D0|h{tzRViBe6qQ|-W{Kt+c*t9 zQ6`hQY4`Lb4Sl;gYrSy!iftGA7c5^An?=Orh3GVjf9_wrvf3SV$G}5)@jwYhu#@|p zC}_IqP`FJrL|1V@z;87kZJbN)>03DLFJ2A^?BUwtSELQ-7Dl58ed3qnyVC;|>|k5O z`ug5*BiaSuhF-`wI2}zvmrIcBWqfA7yQce=^MKj zUCiMuSBQ<#4nFabUU%rv;6H;)Cf)Uo0kk?g*`2PQnHvZ4>N@;mQEeriF5_#CdyjO- zYto^*np5b?<;w|$cFKe8sI!VVUyN@@4M?z!!ToJPN<*iI-ZA|Qen$!F~D zL3g!xW;9sqjXV3&Hr7U8^w1H`Oj`FkNKK0dKhX0>pz{NqXWz7Z7vb64OIyYcpSY}| zv?Ud^C>L!7!l}$zhQp?5^!9MFc6e@xgCf+j3a34gxNRx3L_#Jt;=URkOtKy6R@28E zT5_pJ*ft9HYw+OzB8%as(J`cfc%h1U0+&H}L?dvb?dVf-J;&7qWDsULt1tJSXJfyRp{>($i%EK}? z#Kaq?r!xR+$vV>niS&3b5D^S2Ne3v?u2GTCxNOZ85x(n$Spnv1y1*y%Gfl z93N*usB|exMONmE&)^-y{lgQ4gNuh=i_#@XDJsht4wUi6iv&>0y@~Eap&7=uEL@Ip zU;s+{na`ptxr7U6j$M$drFHC%QzP_Ox5w8h6Gj28^19tLusI4jflH+7v~o|bVp%A;{*}e=DLrQ?XKk`glxB#w20kR z3Ek<+T2OIEmWi;U39xpy)^Ein9LMEmZhwkii zB*ypY#5h5DOpG(2DiY%WicX9ZD6N-Gc%1OeMlnv8&L_tI6ZHSfBUzHNTyq>|-^gYB zq2(KhjL$4UL(NtaKIqqT+(6+}aj9Mq2{F+gk}RUS20iHep);K&0g^F;H*nnUTV5o% zeNCpRBz)>_A>3*%+q4QPo_&jfRV}!kFaN`FiM&6O$kxkb07!4WZx0zF;gQ%c%gplK z^80ndl3*hyEEzN*xJExgCXWPwp%at@LaQYcLK4qv6q0mV9g`b{C;ABw9msUBlG5=o zKdDR02gylY$UdRnas7`T$}x9b9_HO|Ob~`-iKwa*gg@?~{n>DWg}JEV7R521&kn~B z{LYADJhwtF=G<{mRec(_C3WWn{Y_ky9*9I~ML<=y*<51!K-FUUfKE&kOvl7DgC;s% z`aPv4sLIh(07ECH34~TlCd4$J)hMRvveD!kX`zzW=+n@bGrI&y%Ie#=tiECS^+Z-@ z6o#RaX(voTzskEnTxu8EeoSznPviu@~)v#Jc`i>pW z-s?qGH`3SR7%>1AToV#Xs`?kI2h+Fy1d8^4q&Gg@nsSqy+QQRL57h#OPDL#$jwD(qZ=#$lz?AO^rQKK z5@w4wd28|c=r*rn-I~Nszl<#6N4(J({RrI;8H?SAdvmAya6@<3dvpEa#^#h4+e#iF z+(2N<`e>~;uv3tRP%AD@?O!R>mfqW+p1G-astc8&feHe=1bwHywIo&3X_L+&0)WVR zG__Q>m<*Fnpy-!JpV*3ak#$yJb0-$&roG{0H11&Jg$)^gZwuEnKf7i5zia_r6TfS* zbJ?Z?<+7DK5cl6&PK$?!=d&}vY}YR}bxMJUmoxQCT`tygc)4J*GQn5=;!+WlMH_@K z=LbW?en}0&my|A&5NE4oO4z~4&GX&?xPKzo1#Bk+OE`L%P>V}49cvJtp0usgr0Qwkgu zWIDe=vGZHBVuIi+|G^P4S@irCFSRF|De3$M$@44zil5&XqpfSxa~p5~eRS!}QTnI(+TRW?RE;KB> z486d=gtQJL1IeANyKK>?wj$_40qY-wZuz6J=2pG-;<>fn!yrVK4&Qyx(w^DpFWKRw zHzyouxaEr-(&sPP-e4eNbh)HMD&DB9S-kiwen@REMf9A9u8tiT%|#nOG-Y13l2U;F z)Pn{z(joS@hv!)m9;UM%r0pxdA){cMy)*G(>k>M+)Y|3c`9J!v9#bu-@f0@swNeXW z$VqmJ_P8ANa=%!l?p3X)`F8YWk6CGSOG4*jYcmAgz7pA&<{Pp%=q^dEJ+?P!G)n)K zswIi)LVR}%HIPYxQ1r*K-9hJ2wmVBzF}|@y#VXNW^S}I{X3DgDKIjc15pQp}y)>&= zxDgYqxz~`&OrLO0ynZ1Z%aU<`{vx)=ec>LnspfG??rRh!-`|ocxj)=xF>NJ5-2KLF zm;cUn%YF!uBH*S5C3BDe{g++a(4{Hw@MNpIWI9(IQd zh}ykn-vu|m_?CmMYyW9G+D?;7`rya==63Y;?NLid))#zfPtlEMZo!D8_&DLNGgt+e?V28(A&G5AFM*_L{61_4UiWaC~I`9^3b~ zrE-p3ecloGXPeCwa)E6&lT7Q3-(own&1O>2?S&jEc&adkb7{}D|NBDr%2xN}px1f0 zJMQCuO#DB+(EsX%=qFm}P{9KTGP$_2MDs8T&w~q!mi8>H6x`BW;^FtDJqJDXLoJlw zjKDMlROMH>@dP}Rrr<0LO>fGLCIX8ZNuTr3w`%7V4<;bP)>T8f!*Vp+oZcxQ0wGa) z46*jndS`RkKLuH>gyf&`P?}4S5jM@WCX&x;EZfI{6lCg2aczLMktdouqYZy2+95tu zckK@aXnQ0KJ{tJX1b1QEQ8KH?_)lHW0kPnbx$CixwlK-`DNdLHf$HhZ8GN~tI=c|hQia%SRu zQPy+}i4T+noeGVgZlQvD8RNm*a|21`ijW3h##9V-CrW@yg|O^!=fSEEQwMUGrKxN{TTa%~6r%D{RAW$gqlPTM(jut^E{)`ioh{^Q~q3 z65HI3nO0J5@ADqAG6ERQ=28|6nS^i4{tIkNud`l3wH*s43_&EcO5-SDG~ZH|uUImn zxQb$1*M8%R5KnyM4AggYu*13c=+ytCFSh*OhhP83qv(~)2|ehIMRb$#bJDji-M!13 zeT8ZGoukpl;n5H~FJ5LEM+#Dx_AD7E<7uZ$dzP;?O?LC6{V9xjdD<9EF^f?PR-+cJ zM(wp4wa;qQeydRjtVSKQ8g)I4`j&L8f~($U7{6j&1_PK#g9$2d4v&TtIL+okH-iux!yu#hAcH#dIFMvB4&&8k z97e3oIE-1FaTv8WZEf!)N}4D2Qj!#L3#1#uWgiN!FC z5sP7#2m#S)GaJZGWN~YyOa(U^69YF(kVrTrBjLwOQ#eWmQE=$ z3{y%Yjl`W=jl$?ip9^CneH2DU`Y4Qx^idcU>7(p0fmt?vF3hp7Hc9WBAKie+!cG14 z$sD}%4x_KL8=W@qt)S+DJiqy&&F)|h--5HIdBxE%)+jn74brUT@^SR685QDaC_ZnN zLot(C4#k{iITW*-(n6ev*5oI?LF$Mw8u0 zdo1B|YBwi7tEWw;ME2IpKc#tqmmxxXXl-qdYgyfj7lOSIyA;X}l{Tmhz5G+!UJf&L z@>ErOIMSUS*&Kwp`u1wq0aR5+jg|iC^-&+^G_6*(3AO~WVJcq)OOgkx+R5?m@C?jf z8M*QN1WlElj4qsTeA&LWDipy2FnyKq69o0Df*jX{4pNB+dGj#r5+vPks{%vX2r=nfXDUSJX3U{7uu@rmm#*XNu7 z$n%D*UyZ2&2lIU&RyOK?caBc?SG$9wqbZF1R^u9UG}ly{BE88%u|k#pbd zony{@n?dtt`;E+G;lc~Y+0B>g>SOcGFWJq9g_99;Y`z7{`LJbUGvA`+eAxc6nQyP< zeAr5|nQx!veAt$ne*%wjPc<`s)!m{qKXSz)=YF z-VTnn?zBG|!kn+Q9!y1Umho7mW*Lu9L6-5D&$EojoSbDm=G83YF*jxzkNNF28StNN zIL2m%*)cLR49B?4FdU;Y!*Gnr48t=b5}(Wrv*V*@^pxLy?v~Brs^zG8tGO1eMlD*6 z+G`%Q(i`+vrvSat&re4|e^TxMG~T6rG|Xx%}z^I;w}Qex$i{&W@lU~ERso3&*+ z7nOQOwK^JTIU|;fRx@H<(`q0_pyfcz88ND@X2klY)j%vUEC*W7h=q#Pj9BNi8i?hN zG+9_;Z%s4jtNVQYge3)%)=1a9x*nF62 zZ01X~Q`mf%Wo+h4wNu!9m|<+@v*j0j+1bp8^-!B}_%^f~H;%zBgwfxq- zV$4aN(&k!?RExJgTWO!?Rc(JgTWO!?U+M zcvMqmhG$=S@TjH&covLI2&vHc#jt6A18T^_V`S8dk4xWy6^z+kcr=VLraJNnoH1pR zM$yvdA*zDWX8`kb7D&J^buO$X7=6ZoS|KuK)W7Lan9wMq2q}i6btt`?JA%o={yp4W z?@YSum`E_iJ`+hbQ^41ZK9bR}*Eutq!X>aV#l{5hf2>SI;Dbk}PK6bMMkhe{g<&|} zNF+8M;4s>(Y#2yk4)(kWHPLVp@I97ai9k2%w&5ZGyb+qoblhL-v1agdaDlF#g|94{ zpiz8V@fmwwFgTo%Wi46}i2 z(GD)Gy`}DPuiaeO3NT7yDBB%QHb&6-dZUN?8<>r*a5N0OTFqsT^~R$K!L3y>6Rpaz zEmvcs3zk%}Sqhg2(|P={j#`3Rhhy%3HnmWtJMr7q|0%C%T`2?pw3>=E^;b?0lYb(2 zLJGJlX~1(?rhpw~!7G)KvL=P!e@N$plWRjV zE~!%NLT0jJV2~sk*HkID;hMv2N8T@(MBx9lSG*knJ2I!LIx=$JgB|z5Ws#9ADjO&y z%U!8r-8Wt*+<_fVxNcX|%1~Q0P34&J(q}ICW5&xhtkkg?22$N-7!b*37#uO1Vc?0g z83qnEn_=KLvl#~NFq>iE=y@xlVA$9!SpYpfpl zD&+@Q&0E%d1QgMI1Vqq%1bBbrr(RoKb|TYCgm~O)Rnk0M3P{?mYLUY@C7)>9XR2C) z-bw<-PFB4VAgnT~@T_^@!Ri+QZmxb29GIy#bsmm##VcTHRCY!pZyk}JRaC~1C14Mg zE`X6#nYjjM8r+a@%2|qjWrv(ER3)~+Htk?+RYnfH06yH6E`SgA*{WnMP0*j067?%P zfow6@PvuL@SHHwU^-C-wQGq*rrnh=_rT5UCJ)C*?|A|`-T`s7cNBYD5`X+Wyig=N{ zKIm3qyzt3ApdiJfa3?Jjo89v4vOn!W4ZxV$-lSIHD!}pRq#vceOb`q&{0DO0E zV!yIqR$aa$_D!QWHhc?K5nG_?$IA{Y^cpMB=$F*kqlGqyaL+3R)g${>5(5-IX5VT* znOB^bs3akImiTA-@2Srx{T|Nx(#@$pyYzcFyG!>y%yIh9=F#p1`r}FK*P2I=1Cure z?>cP^-eKApyqn}Xp#H;9G$hXf9RS}=3I*buRdRERxaNrBGkw_`H?@vlN zSIOm*xXnLvk>`^C{v)NSM^c0S1c5{b!5jAPlTa%V96l>VYX4`-3W4^kvav!BAeU$U z0ku}^Ck3Ib5csPZ!)1ByZ+{p(p zND~y~AQd9yPvTGE0gh{RLnA$K;cDF&qE11$Lpuq#5+D}wQr#ph__5pcIi}XKc?4iI zWdz_gWdvY%`9-wHK7Y+;GIt8_wK<&PZPwJTr`ab;%&tFtuAure<< zSpxP@=>ix@RfkO$T!l^WA+HQuII4jUc%=*A!(HhD_+VEYHd$~PHmUR?q>&lyYp6_) z@Fgk_n=C9XB85IE>GtSFNnI`qE-1b&=;gpyN0Z+c24oL4Tw|P%cnn^a9Fhz?C;+Eg zH8TW|Vj*DwStw-)X2X;r*ejJX1mB4%>NYYf(>PLL9)Te>kHCPMM_@Q#oy0uFnKFv; z3Y*a|tA#;=Uap&q8!d!j>7y_v(nn!zq>sWF(T>92iqJIB(ie|{S+tmveI9xW(Qz=- zx9RYta-uQAA_Q|^jxWQSOIf}a3y|#1{#_&1#c+w1uOe$%B^@b8Giy0*z`Xu=)9?j8 zoLULMhosJ86g6}fd}F07WRym9794F!nFX_b>JaE@prdR&1oKL(f#iH!4#n8A9Ewq5 zITY_(dr)Nup#GqS!5S@d3Nf3sF-EZ-MsT!ZosF2+3t-kr0Q0aOmME-~HU`6+J|-O= zOtiFlFe9gp!4yp&GmqwV94cr!qMIM4UCj6u!rrg4{t~Qk*kZ6Qyc0_N*B6)sV9+XA zV8)VA(Mxpm!vOfcR*YaXm(0_&E z13-IXJdyqS59ZO;@rmL>oIQgYIbuWcqx)nYz1NG+T2IQ^# zExCJha^?gUk+tLHRHwU&n~;6T8D8*t-%F{pJRWt|R=bnwt>f-mzc-v78lR3d>F;R{ zNHWnHgey+I$6OsRfqE-vx*NDNg_#=l?rh${ulTvh%mQS{)P?TqCkT_8%Tdfb^@ z5`a1ak=)4Qb7x{DK#+{xnFHOS3_&AXG+L8{?SHUcG|9=CV0EnrRKEq>75-pNL6)pRE_ znCiQe1ps7Dc6uiZP;lOGdk;kb)OIIJAjq_+5f_{sPVZ!fidybuacv?uuZrvFtjY@9 zy4nIPoHL~T5J$Q1i%*)&AB7Q4w7a-Jl{;tu59 zt@hi0u{|c|<9mFY zE%t@PI9(1JWnE!lEQ4;t!CEq&TrX(;aY~SG-afk`ycgCR%J`8i1@{4;@kfK^vtoqJ^61VD%dW6d?=t@|;gitHKsCZp{N6!DY@db(I1a4DRfxRFL?wb82v6sF)9)$-RovXVNwSt;#<-PRK$rn zjOTf3kqPkyuAO8xo|7|i(ffD{ISrCCA$J7w_xNV&TE8w7L^1MuP#~<&QE&&c>B!_; zwaW4PS*rH^BJFPc!${vdK<{axP9PqTJf@JgLK`V)4y)~P5T5R|ArLK38v-3M(uP2@ Qi(}e=JlKB^*bwpm0aKCS7XSbN literal 0 HcmV?d00001 diff --git a/dsr/dsr/test/generate_test_data.py b/dsr/dsr/test/generate_test_data.py new file mode 100644 index 00000000..b0b68005 --- /dev/null +++ b/dsr/dsr/test/generate_test_data.py @@ -0,0 +1,28 @@ +"""Generate model parity test case data for DeepSymbolicOptimizer.""" + +from pkg_resources import resource_filename + +from dsr import DeepSymbolicOptimizer + + +# Shorter config run for parity test +CONFIG_TRAINING_OVERRIDE = { + "n_samples" : 1000, + "batch_size" : 100 +} + + +def main(): + + # Train the model + model = DeepSymbolicOptimizer("config.json") + model.config_training.update(CONFIG_TRAINING_OVERRIDE) + model.train() + + # Save the model + save_path = resource_filename("dsr.test", "data/test_model") + model.save(save_path) + + +if __name__ == "__main__": + main() diff --git a/dsr/dsr/test/test_core.py b/dsr/dsr/test/test_core.py new file mode 100644 index 00000000..ebdd55ca --- /dev/null +++ b/dsr/dsr/test/test_core.py @@ -0,0 +1,47 @@ +"""Test cases for DeepSymbolicOptimizer on each Task.""" + +from pkg_resources import resource_filename + +import pytest +import tensorflow as tf +import numpy as np + +from dsr import DeepSymbolicOptimizer +from dsr.test.generate_test_data import CONFIG_TRAINING_OVERRIDE + + +@pytest.fixture +def model(): + return DeepSymbolicOptimizer("config.json") + + +@pytest.fixture +def cached_results(model): + save_path = resource_filename("dsr.test", "data/test_model") + model.load(save_path) + results = model.sess.run(tf.trainable_variables()) + + return results + + +@pytest.mark.parametrize("config", ["config.json"]) +def test_task(model, config): + """Test that Tasks do not crash for various configs.""" + + model.update_config(config) + model.config_training.update({"n_samples" : 10, + "batch_size" : 5 + }) + model.train() + + +def test_model_parity(model, cached_results): + """Compare results to last""" + + model.config_training.update(CONFIG_TRAINING_OVERRIDE) + model.train() + results = model.sess.run(tf.trainable_variables()) + + cached_results = np.concatenate([a.flatten() for a in cached_results]) + results = np.concatenate([a.flatten() for a in results]) + np.testing.assert_array_almost_equal(results, cached_results) diff --git a/dsr/dsr/test/test_prior.py b/dsr/dsr/test/test_prior.py new file mode 100644 index 00000000..b94293fc --- /dev/null +++ b/dsr/dsr/test/test_prior.py @@ -0,0 +1,426 @@ +"""Tests for various Priors.""" + +import pytest + +from dsr.core import DeepSymbolicOptimizer +from dsr.test.generate_test_data import CONFIG_TRAINING_OVERRIDE +from dsr.program import from_tokens, Program +from dsr.memory import Batch +from dsr.controller import parents_siblings + +import numpy as np + + +BATCH_SIZE = 1000 + + +@pytest.fixture +def model(): + return DeepSymbolicOptimizer("config.json") + + +def assert_invalid(model, cases): + cases = [Program.library.actionize(case) for case in cases] + batch = make_batch(model, cases) + logp = model.controller.compute_probs(batch, log=True) + print(batch) + assert all(np.isneginf(logp)), \ + "Found invalid case with probability > 0." + + +def assert_valid(model, cases): + cases = [Program.library.actionize(case) for case in cases] + batch = make_batch(model, cases) + logp = model.controller.compute_probs(batch, log=True) + assert all(logp > -np.inf), \ + "Found valid case with probability 0." + + +def make_sequence(model, L): + """Utility function to generate a sequence of length L""" + X = Program.library.input_tokens[0] + U = Program.library.unary_tokens[0] + B = Program.library.binary_tokens[0] + num_B = (L - 1) // 2 + num_U = int(L % 2 == 0) + num_X = num_B + 1 + case = [B] * num_B + [U] * num_U + [X] * num_X + assert len(case) == L + case = case[:model.controller.max_length] + return case + + +def make_batch(model, actions): + """ + Utility function to generate a Batch from (unfinished) actions. + + This uses essentially the same logic as controller.py's loop_fn, except + actions are prescribed instead of samples. Is there a way to refactor these + with less code reuse? + """ + + batch_size = len(actions) + L = model.controller.max_length + + # Pad actions to maximum length + actions = np.array([np.pad(a, (0, L - len(a)), "constant") + for a in actions], dtype=np.int32) + + # Initialize obs + prev_actions = np.zeros_like(actions) + parents = np.zeros_like(actions) + siblings = np.zeros_like(actions) + + arities = Program.library.arities + parent_adjust = Program.library.parent_adjust + + # Set initial values + empty_parent = np.max(parent_adjust) + 1 + empty_sibling = len(arities) + action = empty_sibling + parent, sibling = empty_parent, empty_sibling + prior = np.array([model.prior.initial_prior()] * batch_size) + + priors = [] + lengths = np.zeros(batch_size, dtype=np.int32) + finished = np.zeros(batch_size, dtype=np.bool_) + dangling = np.ones(batch_size, dtype=np.int32) + for i in range(L): + partial_actions = actions[:, :(i + 1)] + + # Set prior and obs used to generate this action + prev_actions[:, i] = action + parents[:, i] = parent + siblings[:, i] = sibling + priors.append(prior) + + # Compute next obs and prior + action = actions[:, i] + parent, sibling = parents_siblings(tokens=partial_actions, + arities=arities, + parent_adjust=parent_adjust) + dangling += arities[action] - 1 + prior = model.prior(partial_actions, parent, sibling, dangling) + finished = np.where(np.logical_and(dangling == 0, lengths == 0), + True, + False) + lengths = np.where(finished, + i + 1, + lengths) + + lengths = np.where(lengths == 0, L, lengths) + obs = [prev_actions, parents, siblings] + priors = np.array(priors).swapaxes(0, 1) + rewards = np.zeros(batch_size, dtype=np.float32) + batch = Batch(actions, obs, priors, lengths, rewards) + return batch + + +def test_repeat(model): + """Test cases for RepeatConstraint.""" + + model.config_prior = {} # Turn off all other Priors + model.config_prior["repeat"] = { + "tokens" : ["sin", "cos"], + "min_" : None, # Not yet supported + "max_" : 2 + } + model.config_training.update(CONFIG_TRAINING_OVERRIDE) + model.train() + + invalid_cases = [] + invalid_cases.append(["sin"] * 3) + invalid_cases.append(["cos"] * 3) + invalid_cases.append(["sin", "cos", "sin"]) + invalid_cases.append(["mul", "sin"] * 3) + invalid_cases.append(["mul", "sin", "x1", "sin", "mul", "cos"]) + assert_invalid(model, invalid_cases) + + valid_cases = [] + valid_cases.append(["mul"] + ["sin"] * 2 + ["log"] * 2) + valid_cases.append(["sin"] + ["mul", "exp"] * 4 + ["cos"]) + assert_valid(model, valid_cases) + + +def test_descendant(model): + """Test cases for descendant RelationalConstraint.""" + + descendants = "add,mul" + ancestors = "exp,log" + + library = Program.library + model.config_prior = {} # Turn off all other Priors + model.config_prior["relational"] = { + "targets" : descendants, + "effectors" : ancestors, + "relationship" : "descendant" + } + + model.config_training.update(CONFIG_TRAINING_OVERRIDE) + model.train() + + descendants = library.actionize(descendants) + ancestors = library.actionize(ancestors) + + U = [i for i in library.unary_tokens + if i not in ancestors and i not in descendants][0] + B = [i for i in library.binary_tokens + if i not in ancestors and i not in descendants][0] + + # For each D-A combination, generate invalid cases where A is an ancestor + # of D + invalid_cases = [] + for A in ancestors: + for D in descendants: + invalid_cases.append([A, D]) + invalid_cases.append([A] * 10 + [D]) + invalid_cases.append([A] + [U, B] * 5 + [D]) + assert_invalid(model, invalid_cases) + + # For each D-A combination, generate valid cases where A is not an ancestor + # of D + valid_cases = [] + for A in ancestors: + for D in descendants: + valid_cases.append([U, D]) + valid_cases.append([D] + [U] * 10 + [A]) + assert_valid(model, valid_cases) + + +def test_trig(model): + """Test cases for TrigConstraint.""" + + library = Program.library + model.config_prior = {} # Turn off all other Priors + model.config_prior["trig"] = {} + model.config_training.update(CONFIG_TRAINING_OVERRIDE) + model.train() + + X = library.input_tokens[0] + U = [i for i in library.unary_tokens + if i not in library.trig_tokens][0] + B = library.binary_tokens[0] + + # For each trig-trig combination, generate invalid cases where one Token is + # a descendant the other + invalid_cases = [] + trig_tokens = library.trig_tokens + for t1 in trig_tokens: + for t2 in trig_tokens: + invalid_cases.append([t1, t2, X]) # E.g. sin(cos(x)) + invalid_cases.append([t1, B, X, t2, X]) # E.g. sin(x + cos(x)) + invalid_cases.append([t1] + [U] * 10 + [t2, X]) + assert_invalid(model, invalid_cases) + + # For each trig-trig pair, generate valid cases where one Token is the + # sibling the other + valid_cases = [] + for t1 in trig_tokens: + for t2 in trig_tokens: + valid_cases.append([B, U, t1, X, t2, X]) # E.g. log(sin(x)) + cos(x) + valid_cases.append([B, t1, X, t2, X]) # E.g. sin(x) + cos(x) + valid_cases.append([U] + valid_cases[-1]) # E.g. log(sin(x) + cos(x)) + assert_valid(model, valid_cases) + + +def test_child(model): + """Test cases for child RelationalConstraint.""" + + library = Program.library + parents = library.actionize("log,exp,mul") + children = library.actionize("exp,log,sin") + + model.config_prior = {} # Turn off all other Priors + model.config_prior["relational"] = { + "targets" : children, + "effectors" : parents, + "relationship" : "child" + } + model.config_training.update(CONFIG_TRAINING_OVERRIDE) + model.train() + + # For each parent-child pair, generate invalid cases where child is one of + # parent's children. + X = library.input_tokens[0] + assert X not in children, \ + "Error in test case specification. Do not include x1 in children." + invalid_cases = [] + for p, c in zip(parents, children): + arity = library.tokenize(p)[0].arity + for i in range(arity): + before = i + after = arity - i - 1 + case = [p] + [X] * before + [c] + [X] * after + invalid_cases.append(case) + assert_invalid(model, invalid_cases) + + +def test_uchild(model): + """Test cases for uchild RelationalConstraint.""" + + library = Program.library + targets = library.actionize("x1") + effectors = library.actionize("sub,div") # i.e. no x1 - x1 or x1 / x1 + + model.config_prior = {} # Turn off all other Priors + model.config_prior["relational"] = { + "targets" : targets, + "effectors" : effectors, + "relationship" : "uchild" + } + model.config_training.update(CONFIG_TRAINING_OVERRIDE) + model.train() + + # Generate valid test cases + valid_cases = [] + valid_cases.append("mul,x1,x1") + valid_cases.append("sub,x1,sub,x1,sub,x1,sin,x1") + valid_cases.append("sub,sub,sub,x1,sin,x1,x1") + valid_cases.append("sub,sin,x1,sin,x1") + assert_valid(model, valid_cases) + + # Generate invalid test cases + invalid_cases = [] + invalid_cases.append("add,sub,x1,x1,sin,x1") + invalid_cases.append("sin,sub,x1,x1") + invalid_cases.append("sub,sub,sub,x1,x1,x1") + assert_invalid(model, invalid_cases) + + +def test_const(model): + """Test cases for ConstConstraint.""" + + # This test case needs the const Token before creating the model + model.config["task"]["name"] = "Nguyen-1c" + model.pool = model.make_pool() # Resets Program.task with new Task + + library = Program.library + model.config_prior = {} # Turn off all other Priors + model.config_prior["const"] = {} + model.config_training.update(CONFIG_TRAINING_OVERRIDE) + model.train() + + # Generate valid test cases + valid_cases = [] + valid_cases.append("mul,const,x1") + valid_cases.append("sub,const,sub,const,x1") + assert_valid(model, valid_cases) + + # Generate invalid test cases + invalid_cases = [] + invalid_cases.append("sin,const") + invalid_cases.append("mul,const,const") + invalid_cases.append("sin,add,const,const") + assert_invalid(model, invalid_cases) + + +def test_sibling(model): + """Test cases for sibling RelationalConstraint.""" + + library = Program.library + targets = library.actionize("sin,cos") + effectors = library.actionize("x1") + + model.config_prior = {} # Turn off all other Priors + model.config_prior["relational"] = { + "targets" : targets, + "effectors" : effectors, + "relationship" : "sibling" + } + model.config_training.update(CONFIG_TRAINING_OVERRIDE) + model.train() + + # Generate valid test cases + valid_cases = [] + valid_cases.append("mul,sin,x1,cos,x1") + valid_cases.append("sin,cos,x1") + valid_cases.append("add,add,sin,mul,x1,x1,cos,x1,x1") + assert_valid(model, valid_cases) + + # Generate invalid test cases + invalid_cases = [] + invalid_cases.append("add,x1,sin,x1") + invalid_cases.append("add,sin,x1,x1") + invalid_cases.append("add,add,sin,mul,x1,x1,x1,sin,x1") + assert_invalid(model, invalid_cases) + + +def test_inverse(model): + """Test cases for InverseConstraint.""" + + library = Program.library + model.config_prior = {} # Turn off all other Priors + model.config_prior["inverse"] = {} + model.config_training.update(CONFIG_TRAINING_OVERRIDE) + model.train() + + # Generate valid cases + valid_cases = [] + valid_cases.append("exp,sin,log,cos,exp,x1") + valid_cases.append("mul,sin,log,x1,exp,cos,x1") + assert_valid(model, valid_cases) + + # Generate invalid cases for each inverse + invalid_cases = [] + invalid_cases.append("mul,sin,x1,exp,log,x1") + for t1, t2 in library.inverse_tokens.items(): + invalid_cases.append([t1, t2]) + invalid_cases.append([t2, t1]) + assert_invalid(model, invalid_cases) + + +@pytest.mark.parametrize("minmax", [(10, 10), (4, 30), (None, 10), (10, None)]) +def test_length(model, minmax): + """Test cases for LengthConstraint.""" + + min_, max_ = minmax + model.config_prior = {} # Turn off all other Priors + model.config_prior["length"] = {"min_" : min_, "max_" : max_} + model.config_training.update(CONFIG_TRAINING_OVERRIDE) + model.train() + + # First, check that randomly generated samples do not violate constraints + actions, _, _ = model.controller.sample(BATCH_SIZE) + programs = [from_tokens(a, optimize=True) for a in actions] + lengths = [len(p.traversal) for p in programs] + if min_ is not None: + min_L = min(lengths) + assert min_L >= min_, \ + "Found min length {} but constrained to {}.".format(min_L, min_) + if max_ is not None: + max_L = max(lengths) + assert max_L <= max_, \ + "Found max length {} but constrained to {}.".format(max_L, max_) + + # Next, check valid and invalid test cases based on min_ and max_ + # Valid test cases should not be constrained + # Invalid test cases should all be constrained + valid_cases = [] + invalid_cases = [] + + # Initial prior prevents length-1 tokens + case = make_sequence(model, 1) + invalid_cases.append(case) + + if min_ is not None: + # Generate an invalid case that is one Token too short + if min_ > 1: + case = make_sequence(model, min_ - 1) + invalid_cases.append(case) + + # Generate a valid case that is exactly the minimum length + case = make_sequence(model, min_) + valid_cases.append(case) + + if max_ is not None: + # Generate an invalid case that is one Token too long (which will be + # truncated to dangling == 1) + case = make_sequence(model, max_ + 1) + invalid_cases.append(case) + + # Generate a valid case that is exactly the maximum length + case = make_sequence(model, max_) + valid_cases.append(case) + + assert_valid(model, valid_cases) + assert_invalid(model, invalid_cases) diff --git a/dsr/dsr/train.py b/dsr/dsr/train.py new file mode 100644 index 00000000..bd819eb3 --- /dev/null +++ b/dsr/dsr/train.py @@ -0,0 +1,508 @@ +"""Defines main training loop for deep symbolic regression.""" + +import os +import multiprocessing +from itertools import compress +from datetime import datetime +from collections import defaultdict + +import tensorflow as tf +import pandas as pd +import numpy as np + +from dsr.program import Program, from_tokens +from dsr.utils import empirical_entropy, is_pareto_efficient, setup_output_files +from dsr.memory import Batch, make_queue + +# Ignore TensorFlow warnings +os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' +tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR) + +# Set TensorFlow seed +tf.random.set_random_seed(0) + + +# Work for multiprocessing pool: optimize constants and compute reward +def work(p): + optimized_constants = p.optimize() + return optimized_constants, p.base_r + + +def hof_work(p): + return [p.r, p.base_r, p.count, repr(p.sympy_expr), repr(p), p.evaluate] + + +def pf_work(p): + return [p.complexity_eureqa, p.r, p.base_r, p.count, repr(p.sympy_expr), repr(p), p.evaluate] + + +def learn(sess, controller, pool, + logdir="./log", n_epochs=None, n_samples=1e6, + batch_size=1000, complexity="length", complexity_weight=0.001, + const_optimizer="minimize", const_params=None, alpha=0.1, + epsilon=0.01, n_cores_batch=1, verbose=True, summary=True, + output_file=None, save_all_r=False, baseline="ewma_R", + b_jumpstart=True, early_stopping=False, hof=10, eval_all=False, + pareto_front=False, debug=0): + """ + Executes the main training loop. + + Parameters + ---------- + sess : tf.Session + TenorFlow Session object. + + controller : dsr.controller.Controller + Controller object used to generate Programs. + + pool : multiprocessing.Pool or None + Pool to parallelize reward computation. For the control task, each + worker should have its own TensorFlow model. If None, a Pool will be + generated if n_cores_batch > 1. + + logdir : str, optional + Name of log directory. + + n_epochs : int or None, optional + Number of epochs to train when n_samples is None. + + n_samples : int or None, optional + Total number of expressions to sample when n_epochs is None. In this + case, n_epochs = int(n_samples / batch_size). + + batch_size : int, optional + Number of sampled expressions per epoch. + + complexity : str, optional + Complexity penalty name. + + complexity_weight : float, optional + Coefficient for complexity penalty. + + const_optimizer : str or None, optional + Name of constant optimizer. + + const_params : dict, optional + Dict of constant optimizer kwargs. + + alpha : float, optional + Coefficient of exponentially-weighted moving average of baseline. + + epsilon : float or None, optional + Fraction of top expressions used for training. None (or + equivalently, 1.0) turns off risk-seeking. + + n_cores_batch : int, optional + Number of cores to spread out over the batch for constant optimization + and evaluating reward. If -1, uses multiprocessing.cpu_count(). + + verbose : bool, optional + Whether to print progress. + + summary : bool, optional + Whether to write TensorFlow summaries. + + output_file : str, optional + Filename to write results for each iteration. + + save_all_r : bool, optional + Whether to save all rewards for each iteration. + + baseline : str, optional + Type of baseline to use: grad J = (R - b) * grad-log-prob(expression). + Choices: + (1) "ewma_R" : b = EWMA() + (2) "R_e" : b = R_e + (3) "ewma_R_e" : b = EWMA(R_e) + (4) "combined" : b = R_e + EWMA( - R_e) + In the above, is the sample average _after_ epsilon sub-sampling and + R_e is the (1-epsilon)-quantile estimate. + + b_jumpstart : bool, optional + Whether EWMA part of the baseline starts at the average of the first + iteration. If False, the EWMA starts at 0.0. + + early_stopping : bool, optional + Whether to stop early if stopping criteria is reached. + + hof : int or None, optional + If not None, number of top Programs to evaluate after training. + + eval_all : bool, optional + If True, evaluate all Programs. While expensive, this is useful for + noisy data when you can't be certain of success solely based on reward. + If False, only the top Program is evaluated each iteration. + + pareto_front : bool, optional + If True, compute and save the Pareto front at the end of training. + + debug : int, optional + Debug level, also passed to Controller. 0: No debug. 1: Print initial + parameter means. 2: Print parameter means each step. + + Returns + ------- + result : dict + A dict describing the best-fit expression (determined by base_r). + """ + + # Config assertions and warnings + assert n_samples is None or n_epochs is None, "At least one of 'n_samples' or 'n_epochs' must be None." + + # Create the summary writer + if summary: + timestamp = datetime.now().strftime("%Y-%m-%d-%H%M%S") + summary_dir = os.path.join("summary", timestamp) + writer = tf.summary.FileWriter(summary_dir, sess.graph) + + # Create log file + if output_file is not None: + all_r_output_file, hof_output_file, pf_output_file = setup_output_files(logdir, output_file) + else: + all_r_output_file = hof_output_file = pf_output_file = None + + # Set the complexity functions + Program.set_complexity_penalty(complexity, complexity_weight) + + # Set the constant optimizer + const_params = const_params if const_params is not None else {} + Program.set_const_optimizer(const_optimizer, **const_params) + + # Initialize compute graph + sess.run(tf.global_variables_initializer()) + + if debug: + tvars = tf.trainable_variables() + def print_var_means(): + tvars_vals = sess.run(tvars) + for var, val in zip(tvars, tvars_vals): + print(var.name, "mean:", val.mean(),"var:", val.var()) + + # Create the pool of workers, if pool is not already given + if pool is None: + if n_cores_batch == -1: + n_cores_batch = multiprocessing.cpu_count() + if n_cores_batch > 1: + pool = multiprocessing.Pool(n_cores_batch) + + # Create the priority queue + k = controller.pqt_k + if controller.pqt and k is not None and k > 0: + priority_queue = make_queue(priority=True, capacity=k) + else: + priority_queue = None + + if debug >= 1: + print("\nInitial parameter means:") + print_var_means() + + base_r_history = None + + # Main training loop + p_final = None + base_r_best = -np.inf + r_best = -np.inf + prev_r_best = None + prev_base_r_best = None + ewma = None if b_jumpstart else 0.0 # EWMA portion of baseline + n_epochs = n_epochs if n_epochs is not None else int(n_samples / batch_size) + all_r = np.zeros(shape=(n_epochs, batch_size), dtype=np.float32) + + for step in range(n_epochs): + + # Set of str representations for all Programs ever seen + s_history = set(Program.cache.keys()) + + # Sample batch of expressions from controller + # Shape of actions: (batch_size, max_length) + # Shape of obs: [(batch_size, max_length)] * 3 + # Shape of priors: (batch_size, max_length, n_choices) + actions, obs, priors = controller.sample(batch_size) + + # Instantiate, optimize, and evaluate expressions + if pool is None: + programs = [from_tokens(a, optimize=True) for a in actions] + else: + # To prevent interfering with the cache, un-optimized programs are + # first generated serially. Programs that need optimizing are + # optimized optimized in parallel. Since multiprocessing operates on + # copies of programs, we manually set the optimized constants and + # base reward after the pool joins. + programs = [from_tokens(a, optimize=False) for a in actions] + + # Filter programs that have not yet computed base_r + programs_to_optimize = list(set([p for p in programs if "base_r" not in p.__dict__])) + + # Optimize and compute base_r + results = pool.map(work, programs_to_optimize) + for (optimized_constants, base_r), p in zip(results, programs_to_optimize): + p.set_constants(optimized_constants) + p.base_r = base_r + + # Retrieve metrics + base_r = np.array([p.base_r for p in programs]) + r = np.array([p.r for p in programs]) + l = np.array([len(p.traversal) for p in programs]) + s = [p.str for p in programs] # Str representations of Programs + invalid = np.array([p.invalid for p in programs], dtype=bool) + all_r[step] = base_r + + if eval_all: + success = [p.evaluate.get("success") for p in programs] + # Check for success before risk-seeking, but don't break until after + if any(success): + p_final = programs[success.index(True)] + + # Update reward history + if base_r_history is not None: + for p in programs: + key = p.str + if key in base_r_history: + base_r_history[key].append(p.base_r) + else: + base_r_history[key] = [p.base_r] + + # Collect full-batch statistics + base_r_max = np.max(base_r) + base_r_best = max(base_r_max, base_r_best) + base_r_avg_full = np.mean(base_r) + r_max = np.max(r) + r_best = max(r_max, r_best) + r_avg_full = np.mean(r) + l_avg_full = np.mean(l) + a_ent_full = np.mean(np.apply_along_axis(empirical_entropy, 0, actions)) + n_unique_full = len(set(s)) + n_novel_full = len(set(s).difference(s_history)) + invalid_avg_full = np.mean(invalid) + + # Risk-seeking policy gradient: train on top epsilon fraction of samples + if epsilon is not None and epsilon < 1.0: + quantile = np.quantile(r, 1 - epsilon, interpolation="higher") + keep = base_r >= quantile + base_r = base_r[keep] + r_train = r = r[keep] + programs = list(compress(programs, keep)) + l = l[keep] + s = list(compress(s, keep)) + invalid = invalid[keep] + actions = actions[keep, :] + obs = [o[keep, :] for o in obs] + priors = priors[keep, :, :] + + # Clip bounds of rewards to prevent NaNs in gradient descent + r = np.clip(r, -1e6, 1e6) + + # Compute baseline + if baseline == "ewma_R": + ewma = np.mean(r) if ewma is None else alpha*np.mean(r) + (1 - alpha)*ewma + b_train = ewma + elif baseline == "R_e": # Default + ewma = -1 + b_train = quantile + + # Collect sub-batch statistics and write output + if output_file is not None: + base_r_avg_sub = np.mean(base_r) + r_avg_sub = np.mean(r) + l_avg_sub = np.mean(l) + a_ent_sub = np.mean(np.apply_along_axis(empirical_entropy, 0, actions)) + n_unique_sub = len(set(s)) + n_novel_sub = len(set(s).difference(s_history)) + invalid_avg_sub = np.mean(invalid) + stats = np.array([[ + base_r_best, + base_r_max, + base_r_avg_full, + base_r_avg_sub, + r_best, + r_max, + r_avg_full, + r_avg_sub, + l_avg_full, + l_avg_sub, + ewma, + n_unique_full, + n_unique_sub, + n_novel_full, + n_novel_sub, + a_ent_full, + a_ent_sub, + invalid_avg_full, + invalid_avg_sub + ]], dtype=np.float32) + with open(os.path.join(logdir, output_file), 'ab') as f: + np.savetxt(f, stats, delimiter=',') + + # Compute sequence lengths + lengths = np.array([min(len(p.traversal), controller.max_length) + for p in programs], dtype=np.int32) + + # Create the Batch + sampled_batch = Batch(actions=actions, obs=obs, priors=priors, + lengths=lengths, rewards=r) + + # Update and sample from the priority queue + if priority_queue is not None: + priority_queue.push_best(sampled_batch, programs) + pqt_batch = priority_queue.sample_batch(controller.pqt_batch_size) + else: + pqt_batch = None + + # Train the controller + summaries = controller.train_step(b_train, sampled_batch, pqt_batch) + if summary: + writer.add_summary(summaries, step) + writer.flush() + + # Update new best expression + new_r_best = False + new_base_r_best = False + + if prev_r_best is None or r_max > prev_r_best: + new_r_best = True + p_r_best = programs[np.argmax(r)] + + if prev_base_r_best is None or base_r_max > prev_base_r_best: + new_base_r_best = True + p_base_r_best = programs[np.argmax(base_r)] + + prev_r_best = r_best + prev_base_r_best = base_r_best + + # Print new best expression + if verbose: + if new_r_best and new_base_r_best: + if p_r_best == p_base_r_best: + print("\nNew best overall") + p_r_best.print_stats() + else: + print("\nNew best reward") + p_r_best.print_stats() + print("...and new best base reward") + p_base_r_best.print_stats() + + elif new_r_best: + print("\nNew best reward") + p_r_best.print_stats() + + elif new_base_r_best: + print("\nNew best base reward") + p_base_r_best.print_stats() + + # Stop if early stopping criteria is met + if eval_all and any(success): + all_r = all_r[:(step + 1)] + print("Early stopping criteria met; breaking early.") + break + if early_stopping and p_base_r_best.evaluate.get("success"): + all_r = all_r[:(step + 1)] + print("Early stopping criteria met; breaking early.") + break + + if verbose and step > 0 and step % 10 == 0: + print("Completed {} steps".format(step)) + + if debug >= 2: + print("\nParameter means after step {} of {}:".format(step+1, n_epochs)) + print_var_means() + + if save_all_r: + with open(all_r_output_file, 'ab') as f: + np.save(f, all_r) + + # Save the hall of fame + if hof is not None and hof > 0: + programs = list(Program.cache.values()) # All unique Programs found during training + + base_r = [p.base_r for p in programs] + i_hof = np.argsort(base_r)[-hof:][::-1] # Indices of top hof Programs + hof = [programs[i] for i in i_hof] + + if verbose: + print("Evaluating the hall of fame...") + if pool is not None: + results = pool.map(hof_work, hof) + else: + results = list(map(hof_work, hof)) + + eval_keys = list(results[0][-1].keys()) + columns = ["r", "base_r", "count", "expression", "traversal"] + eval_keys + hof_results = [result[:-1] + [result[-1][k] for k in eval_keys] for result in results] + df = pd.DataFrame(hof_results, columns=columns) + if hof_output_file is not None: + print("Saving Hall of Fame to {}".format(hof_output_file)) + df.to_csv(hof_output_file, header=True, index=False) + + # Print error statistics of the cache + n_invalid = 0 + error_types = defaultdict(lambda : 0) + error_nodes = defaultdict(lambda : 0) + for p in Program.cache.values(): + if p.invalid: + n_invalid += p.count + error_types[p.error_type] += p.count + error_nodes[p.error_node] += p.count + if n_invalid > 0: + total_samples = (step + 1)*batch_size # May be less than n_samples if breaking early + print("Invalid expressions: {} of {} ({:.1%}).".format(n_invalid, total_samples, n_invalid/total_samples)) + print("Error type counts:") + for error_type, count in error_types.items(): + print(" {}: {} ({:.1%})".format(error_type, count, count/n_invalid)) + print("Error node counts:") + for error_node, count in error_nodes.items(): + print(" {}: {} ({:.1%})".format(error_node, count, count/n_invalid)) + + # Print the priority queue at the end of training + if verbose and priority_queue is not None: + for i, item in enumerate(priority_queue.iter_in_order()): + print("\nPriority queue entry {}:".format(i)) + p = Program.cache[item[0]] + p.print_stats() + + # Compute the pareto front + if pareto_front: + if verbose: + print("Evaluating the pareto front...") + all_programs = list(Program.cache.values()) + costs = np.array([(p.complexity_eureqa, -p.r) for p in all_programs]) + pareto_efficient_mask = is_pareto_efficient(costs) # List of bool + pf = list(compress(all_programs, pareto_efficient_mask)) + pf.sort(key=lambda p : p.complexity_eureqa) # Sort by complexity + + if pool is not None: + results = pool.map(pf_work, pf) + else: + results = list(map(pf_work, pf)) + + eval_keys = list(results[0][-1].keys()) + columns = ["complexity", "r", "base_r", "count", "expression", "traversal"] + eval_keys + pf_results = [result[:-1] + [result[-1][k] for k in eval_keys] for result in results] + df = pd.DataFrame(pf_results, columns=columns) + if pf_output_file is not None: + print("Saving Pareto Front to {}".format(pf_output_file)) + df.to_csv(pf_output_file, header=True, index=False) + + # Look for a success=True case within the Pareto front + for p in pf: + if p.evaluate.get("success"): + p_final = p + break + + # Close the pool + if pool is not None: + pool.close() + + # Return statistics of best Program + p = p_final if p_final is not None else p_base_r_best + result = { + "r" : p.r, + "base_r" : p.base_r, + } + result.update(p.evaluate) + result.update({ + "expression" : repr(p.sympy_expr), + "traversal" : repr(p), + "program" : p + }) + + return result diff --git a/dsr/dsr/utils.py b/dsr/dsr/utils.py new file mode 100644 index 00000000..1c8113b4 --- /dev/null +++ b/dsr/dsr/utils.py @@ -0,0 +1,154 @@ +"""Utility functions used in deep symbolic regression.""" + +import os +import functools +import numpy as np + + +def is_float(s): + """Determine whether str can be cast to float.""" + + try: + float(s) + return True + except ValueError: + return False + + +# Adapted from: https://stackoverflow.com/questions/32791911/fast-calculation-of-pareto-front-in-python +def is_pareto_efficient(costs): + """ + Find the pareto-efficient points given an array of costs. + + Parameters + ---------- + + costs : np.ndarray + Array of shape (n_points, n_costs). + + Returns + ------- + + is_efficient_maek : np.ndarray (dtype:bool) + Array of which elements in costs are pareto-efficient. + """ + + is_efficient = np.arange(costs.shape[0]) + n_points = costs.shape[0] + next_point_index = 0 # Next index in the is_efficient array to search for + while next_point_index < len(costs): + nondominated_point_mask = np.any(costs < costs[next_point_index], axis=1) + nondominated_point_mask[next_point_index] = True + is_efficient = is_efficient[nondominated_point_mask] # Remove dominated points + costs = costs[nondominated_point_mask] + next_point_index = np.sum(nondominated_point_mask[:next_point_index]) + 1 + is_efficient_mask = np.zeros(n_points, dtype=bool) + is_efficient_mask[is_efficient] = True + return is_efficient_mask + + +def setup_output_files(logdir, output_file): + """ + Writes the main output file header and returns the reward, hall of fame, and Pareto front config filenames. + + Parameters: + ----------- + + logdir : string + Directory to log to. + + output_file : string + Name of output file. + + Returns: + -------- + + all_r_output_file : string + all_r output filename + + hof_output_file : string + hof output filename + + pf_output_file : string + pf output filename + """ + os.makedirs(logdir, exist_ok=True) + output_file = os.path.join(logdir, output_file) + prefix, _ = os.path.splitext(output_file) + all_r_output_file = "{}_all_r.npy".format(prefix) + hof_output_file = "{}_hof.csv".format(prefix) + pf_output_file = "{}_pf.csv".format(prefix) + with open(output_file, 'w') as f: + # r_best : Maximum across all iterations so far + # r_max : Maximum across this iteration's batch + # r_avg_full : Average across this iteration's full batch (before taking epsilon subset) + # r_avg_sub : Average across this iteration's epsilon-subset batch + # n_unique_* : Number of unique Programs in batch + # n_novel_* : Number of never-before-seen Programs per batch + # a_ent_* : Empirical positional entropy across sequences averaged over positions + # invalid_avg_* : Fraction of invalid Programs per batch + headers = ["base_r_best", + "base_r_max", + "base_r_avg_full", + "base_r_avg_sub", + "r_best", + "r_max", + "r_avg_full", + "r_avg_sub", + "l_avg_full", + "l_avg_sub", + "ewma", + "n_unique_full", + "n_unique_sub", + "n_novel_full", + "n_novel_sub", + "a_ent_full", + "a_ent_sub", + "invalid_avg_full", + "invalid_avg_sub"] + f.write("{}\n".format(",".join(headers))) + + return all_r_output_file, hof_output_file, pf_output_file + + +class cached_property(object): + """ + Decorator used for lazy evaluation of an object attribute. The property + should be non-mutable, since it replaces itself. + """ + + def __init__(self, getter): + self.getter = getter + + functools.update_wrapper(self, getter) + + def __get__(self, obj, cls): + if obj is None: + return self + + value = self.getter(obj) + setattr(obj, self.getter.__name__, value) + return value + + +# Entropy computation in batch +def empirical_entropy(labels): + + n_labels = len(labels) + + if n_labels <= 1: + return 0 + + value, counts = np.unique(labels, return_counts=True) + probs = counts / n_labels + n_classes = np.count_nonzero(probs) + + if n_classes <= 1: + return 0 + + ent = 0. + # Compute entropy + for i in probs: + ent -= i * np.log(i) + + return ent diff --git a/dsr/setup.py b/dsr/setup.py new file mode 100644 index 00000000..e192d213 --- /dev/null +++ b/dsr/setup.py @@ -0,0 +1,16 @@ +from distutils.core import setup +from Cython.Build import cythonize +import numpy +import os + +# To build cython code using setup try: +# python setup.py build_ext --inplace + +setup( name='dsr', + version='1.0dev', + description='Deep symbolic regression.', + author='LLNL', + packages=['dsr'], + ext_modules=cythonize([os.path.join('dsr','cyfunc.pyx')]), + include_dirs=[numpy.get_include()] + ) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..c00628f1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +pytest +cython +numpy +tensorflow==1.14 +numba +sympy +pandas +scikit-learn +click +mpi4py +dataclasses +