@@ -28,9 +28,7 @@ Usage:
Every sess.run that requires any of these tensors will sample a new
transition.
-## Methods
-
-
```python
__init__(
@@ -43,8 +41,7 @@ Initializes WrappedPrioritizedReplayBuffer.
#### Args:
-*
: tuple or int. If int, the observation is assumed
- to be a 2D square with sides equal to observation_shape.
+*
: bool, when True it would use a staging area to
prefetch the next sampling batch.
@@ -59,12 +56,20 @@ Initializes WrappedPrioritizedReplayBuffer.
sample_transition_batch.
*
: np.dtype, type of the observations. Defaults to
np.uint8 for Atari 2600.
+*
: tuple of ints, the shape for the action vector. Empty
+ tuple means the action is a scalar.
+*
: tuple of ints, the shape of the reward vector. Empty
+ tuple means the reward is a scalar.
+*
```python
diff --git a/docs/api_docs/python/rainbow_agent.md b/docs/api_docs/python/rainbow_agent.md
index a798a854..96924104 100644
--- a/docs/api_docs/python/rainbow_agent.md
+++ b/docs/api_docs/python/rainbow_agent.md
@@ -1,6 +1,6 @@
# Module: rainbow_agent
diff --git a/docs/api_docs/python/rainbow_agent/RainbowAgent.md b/docs/api_docs/python/rainbow_agent/RainbowAgent.md
index bac36567..220d164c 100644
--- a/docs/api_docs/python/rainbow_agent/RainbowAgent.md
+++ b/docs/api_docs/python/rainbow_agent/RainbowAgent.md
@@ -1,6 +1,6 @@
-
+
@@ -17,9 +17,7 @@ Inherits From: [`DQNAgent`](../dqn_agent/DQNAgent.md)
A compact implementation of a simplified Rainbow agent.
-## Methods
-
-
__init__
+
__init__
```python
__init__(
@@ -35,6 +33,16 @@ Initializes the agent and constructs the components of its graph.
*
`sess` : `tf.Session`, for executing ops.
*
`num_actions` : int, number of actions the agent can take at any
state.
+*
`observation_shape` : tuple of ints or an int. If single int, the
+ observation is assumed to be a 2D square.
+*
`observation_dtype` : tf.DType, specifies the type of the
+ observations. Note that if your inputs are continuous, you should set this
+ to tf.float32.
+*
`stack_size` : int, number of frames to use in state stack.
+*
`network` : function expecting three parameters: (num_actions,
+ network_type, state). This function will return the network_type object
+ containing the tensors output by the network. See
+ dopamine.discrete_domains.atari_lib.rainbow_network as an example.
*
`num_atoms` : int, the number of buckets of the value function
distribution.
*
`vmax` : float, the value distribution support is [-vmax, vmax].
@@ -59,6 +67,12 @@ Initializes the agent and constructs the components of its graph.
*
`use_staging` : bool, when True use a staging area to prefetch the
next training batch, speeding training up by about 30%.
*
`optimizer` : `tf.train.Optimizer`, for training the value function.
+*
`summary_writer` : SummaryWriter object for outputting training
+ statistics. Summary writing disabled if set to None.
+*
`summary_writing_frequency` : int, frequency with which summaries will
+ be written. Lower values will result in slower training.
+
+## Methods
begin_episode
diff --git a/docs/api_docs/python/rainbow_agent/project_distribution.md b/docs/api_docs/python/rainbow_agent/project_distribution.md
index f5d23aef..f5d8556b 100644
--- a/docs/api_docs/python/rainbow_agent/project_distribution.md
+++ b/docs/api_docs/python/rainbow_agent/project_distribution.md
@@ -1,6 +1,6 @@
# rainbow_agent.project_distribution
diff --git a/docs/api_docs/python/run_experiment.md b/docs/api_docs/python/run_experiment.md
index 64a7c2f0..e91d3033 100644
--- a/docs/api_docs/python/run_experiment.md
+++ b/docs/api_docs/python/run_experiment.md
@@ -1,16 +1,23 @@
# Module: run_experiment
-Module defining classes and helper methods for running Atari 2600 agents.
+Module defining classes and helper methods for general agents.
## Classes
-[`class Runner`](./run_experiment/Runner.md): Object that handles running Atari
-2600 experiments.
+[`class Runner`](./run_experiment/Runner.md): Object that handles running
+Dopamine experiments.
[`class TrainRunner`](./run_experiment/TrainRunner.md): Object that handles
-running Atari 2600 experiments.
+running experiments.
+
+## Functions
+
+[`create_agent(...)`](./run_experiment/create_agent.md): Creates an agent.
+
+[`create_runner(...)`](./run_experiment/create_runner.md): Creates an experiment
+Runner.
diff --git a/docs/api_docs/python/run_experiment/Runner.md b/docs/api_docs/python/run_experiment/Runner.md
index 6a6dfeeb..77863a84 100644
--- a/docs/api_docs/python/run_experiment/Runner.md
+++ b/docs/api_docs/python/run_experiment/Runner.md
@@ -1,6 +1,6 @@
@@ -9,7 +9,7 @@
## Class `Runner`
-Object that handles running Atari 2600 experiments.
+Object that handles running Dopamine experiments.
Here we use the term 'experiment' to mean simulating interactions between the
agent and the environment and reporting some statistics pertaining to these
@@ -18,16 +18,15 @@ interactions.
A simple scenario to train a DQN agent is as follows:
```python
+import dopamine.discrete_domains.atari_lib
base_dir = '/tmp/simple_example'
def create_agent(sess, environment):
return dqn_agent.DQNAgent(sess, num_actions=environment.action_space.n)
-runner = Runner(base_dir, create_agent, game_name='Pong')
+runner = Runner(base_dir, create_agent, atari_lib.create_atari_environment)
runner.run()
```
-## Methods
-
-
__init__
+
__init__
```python
__init__(
@@ -43,12 +42,9 @@ Initialize the Runner object in charge of running a full experiment.
*
`base_dir` : str, the base directory to host all required
sub-directories.
*
`create_agent_fn` : A function that takes as args a Tensorflow session
- and an Atari 2600 Gym environment, and returns an agent.
-*
`create_environment_fn` : A function which receives a game name and
- creates an Atari 2600 Gym environment.
-*
`game_name` : str, name of the Atari 2600 domain to run (required).
-*
`sticky_actions` : bool, whether to enable sticky actions in the
- environment.
+ and an environment, and returns an agent.
+*
`create_environment_fn` : A function which receives a problem name and
+ creates a Gym environment for that problem (e.g. an Atari 2600 game).
*
`checkpoint_file_prefix` : str, the prefix to use for checkpoint
files.
*
`logging_file_prefix` : str, prefix to use for the log files.
@@ -65,6 +61,8 @@ Initialize a `tf.Session`. - Initialize a logger. - Initialize an agent. -
Reload from the latest checkpoint, if available, and initialize the Checkpointer
object.
+## Methods
+
run_experiment
```python
diff --git a/docs/api_docs/python/run_experiment/TrainRunner.md b/docs/api_docs/python/run_experiment/TrainRunner.md
index f856191c..813053e4 100644
--- a/docs/api_docs/python/run_experiment/TrainRunner.md
+++ b/docs/api_docs/python/run_experiment/TrainRunner.md
@@ -1,6 +1,6 @@
@@ -11,15 +11,13 @@
Inherits From: [`Runner`](../run_experiment/Runner.md)
-Object that handles running Atari 2600 experiments.
+Object that handles running experiments.
The `TrainRunner` differs from the base `Runner` class in that it does not the
evaluation phase. Checkpointing and logging for the train phase are preserved as
before.
-## Methods
-
-
__init__
+
__init__
```python
__init__(
@@ -35,7 +33,11 @@ Initialize the TrainRunner object in charge of running a full experiment.
*
`base_dir` : str, the base directory to host all required
sub-directories.
*
`create_agent_fn` : A function that takes as args a Tensorflow session
- and an Atari 2600 Gym environment, and returns an agent.
+ and an environment, and returns an agent.
+*
`create_environment_fn` : A function which receives a problem name and
+ creates a Gym environment for that problem (e.g. an Atari 2600 game).
+
+## Methods
run_experiment
diff --git a/docs/api_docs/python/run_experiment/create_agent.md b/docs/api_docs/python/run_experiment/create_agent.md
new file mode 100644
index 00000000..aed497b6
--- /dev/null
+++ b/docs/api_docs/python/run_experiment/create_agent.md
@@ -0,0 +1,34 @@
+
+
+
+# run_experiment.create_agent
+
+```python
+run_experiment.create_agent(
+ *args,
+ **kwargs
+)
+```
+
+Creates an agent.
+
+#### Args:
+
+*
`sess` : A `tf.Session` object for running associated ops.
+*
`environment` : An Atari 2600 Gym environment.
+*
`agent_name` : str, name of the agent to create.
+*
`summary_writer` : A Tensorflow summary writer to pass to the agent
+ for in-agent training statistics in Tensorboard.
+*
`debug_mode` : bool, whether to output Tensorboard summaries. If set
+ to true, the agent will output in-episode statistics to Tensorboard.
+ Disabled by default as this results in slower training.
+
+#### Returns:
+
+*
`agent` : An RL agent.
+
+#### Raises:
+
+*
`ValueError` : If `agent_name` is not in supported list.
diff --git a/docs/api_docs/python/run_experiment/create_runner.md b/docs/api_docs/python/run_experiment/create_runner.md
new file mode 100644
index 00000000..64c8efbf
--- /dev/null
+++ b/docs/api_docs/python/run_experiment/create_runner.md
@@ -0,0 +1,28 @@
+
+
+
+# run_experiment.create_runner
+
+```python
+run_experiment.create_runner(
+ *args,
+ **kwargs
+)
+```
+
+Creates an experiment Runner.
+
+#### Args:
+
+*
`base_dir` : str, base directory for hosting all subdirectories.
+*
`schedule` : string, which type of Runner to use.
+
+#### Returns:
+
+*
`runner` : A `Runner` like object.
+
+#### Raises:
+
+*
`ValueError` : When an unknown schedule is encountered.
diff --git a/docs/api_docs/python/train.md b/docs/api_docs/python/train.md
index c2ca0eff..4d757e07 100644
--- a/docs/api_docs/python/train.md
+++ b/docs/api_docs/python/train.md
@@ -1,18 +1,9 @@
# Module: train
-The entry point for running an agent on an Atari 2600 domain.
+The entry point for running a Dopamine agent.
-
-## Functions
-
-[`create_agent(...)`](./train/create_agent.md): Creates a DQN agent.
-
-[`create_runner(...)`](./train/create_runner.md): Creates an experiment Runner.
-
-[`launch_experiment(...)`](./train/launch_experiment.md): Launches the
-experiment.
diff --git a/docs/api_docs/python/utils.md b/docs/api_docs/python/utils.md
index a1860bc4..0f8fc97f 100644
--- a/docs/api_docs/python/utils.md
+++ b/docs/api_docs/python/utils.md
@@ -1,13 +1,13 @@
# Module: utils
This provides utilities for dealing with Dopamine data.
-See: dopamine//common/logger.py .
+See: dopamine/common/logger.py .
## Functions
diff --git a/docs/api_docs/python/utils/get_latest_file.md b/docs/api_docs/python/utils/get_latest_file.md
index 073b6db0..572d3203 100644
--- a/docs/api_docs/python/utils/get_latest_file.md
+++ b/docs/api_docs/python/utils/get_latest_file.md
@@ -1,6 +1,6 @@
# utils.get_latest_file
diff --git a/docs/api_docs/python/utils/get_latest_iteration.md b/docs/api_docs/python/utils/get_latest_iteration.md
index 340c2124..5d27e3e4 100644
--- a/docs/api_docs/python/utils/get_latest_iteration.md
+++ b/docs/api_docs/python/utils/get_latest_iteration.md
@@ -1,6 +1,6 @@
# utils.get_latest_iteration
diff --git a/docs/api_docs/python/utils/load_baselines.md b/docs/api_docs/python/utils/load_baselines.md
index 6bbba8e9..7a4c9ac2 100644
--- a/docs/api_docs/python/utils/load_baselines.md
+++ b/docs/api_docs/python/utils/load_baselines.md
@@ -1,6 +1,6 @@
# utils.load_baselines
diff --git a/docs/api_docs/python/utils/load_statistics.md b/docs/api_docs/python/utils/load_statistics.md
index 35108a0b..92b2ff4f 100644
--- a/docs/api_docs/python/utils/load_statistics.md
+++ b/docs/api_docs/python/utils/load_statistics.md
@@ -1,6 +1,6 @@
# utils.load_statistics
diff --git a/docs/api_docs/python/utils/read_experiment.md b/docs/api_docs/python/utils/read_experiment.md
index 70cf48c8..48203354 100644
--- a/docs/api_docs/python/utils/read_experiment.md
+++ b/docs/api_docs/python/utils/read_experiment.md
@@ -1,6 +1,6 @@
# utils.read_experiment
diff --git a/docs/api_docs/python/utils/summarize_data.md b/docs/api_docs/python/utils/summarize_data.md
index 50ebbd8c..673b5497 100644
--- a/docs/api_docs/python/utils/summarize_data.md
+++ b/docs/api_docs/python/utils/summarize_data.md
@@ -1,6 +1,6 @@
# utils.summarize_data
diff --git a/dopamine/agents/dqn/configs/dqn.gin b/dopamine/agents/dqn/configs/dqn.gin
index 06c3427f..13b5727d 100644
--- a/dopamine/agents/dqn/configs/dqn.gin
+++ b/dopamine/agents/dqn/configs/dqn.gin
@@ -1,7 +1,8 @@
# Hyperparameters follow the classic Nature DQN, but we modify as necessary to
# match those used in Rainbow (Hessel et al., 2018), to ensure apples-to-apples
# comparison.
-import dopamine.atari.run_experiment
+import dopamine.discrete_domains.atari_lib
+import dopamine.discrete_domains.run_experiment
import dopamine.agents.dqn.dqn_agent
import dopamine.replay_memory.circular_replay_buffer
import gin.tf.external_configurables
@@ -23,9 +24,10 @@ tf.train.RMSPropOptimizer.momentum = 0.0
tf.train.RMSPropOptimizer.epsilon = 0.00001
tf.train.RMSPropOptimizer.centered = True
-Runner.game_name = 'Pong'
+atari_lib.create_atari_environment.game_name = 'Pong'
# Sticky actions with probability 0.25, as suggested by (Machado et al., 2017).
-Runner.sticky_actions = True
+atari_lib.create_atari_environment.sticky_actions = True
+create_agent.agent_name = 'dqn'
Runner.num_iterations = 200
Runner.training_steps = 250000 # agent steps
Runner.evaluation_steps = 125000 # agent steps
diff --git a/dopamine/agents/dqn/configs/dqn_acrobot.gin b/dopamine/agents/dqn/configs/dqn_acrobot.gin
new file mode 100644
index 00000000..4cf06168
--- /dev/null
+++ b/dopamine/agents/dqn/configs/dqn_acrobot.gin
@@ -0,0 +1,35 @@
+# Hyperparameters for a simple DQN-style Acrobot agent. The hyperparameters
+# chosen achieve reasonable performance.
+import dopamine.discrete_domains.gym_lib
+import dopamine.discrete_domains.run_experiment
+import dopamine.agents.dqn.dqn_agent
+import dopamine.replay_memory.circular_replay_buffer
+import gin.tf.external_configurables
+
+DQNAgent.observation_shape = %gym_lib.ACROBOT_OBSERVATION_SHAPE
+DQNAgent.observation_dtype = %gym_lib.ACROBOT_OBSERVATION_DTYPE
+DQNAgent.stack_size = %gym_lib.ACROBOT_STACK_SIZE
+DQNAgent.network = @gym_lib.acrobot_dqn_network
+DQNAgent.gamma = 0.99
+DQNAgent.update_horizon = 1
+DQNAgent.min_replay_history = 500
+DQNAgent.update_period = 4
+DQNAgent.target_update_period = 100
+DQNAgent.epsilon_fn = @dqn_agent.identity_epsilon
+DQNAgent.tf_device = '/gpu:0' # use '/cpu:*' for non-GPU version
+DQNAgent.optimizer = @tf.train.AdamOptimizer()
+
+tf.train.AdamOptimizer.learning_rate = 0.001
+tf.train.AdamOptimizer.epsilon = 0.0003125
+
+create_gym_environment.environment_name = 'Acrobot'
+create_gym_environment.version = 'v1'
+create_agent.agent_name = 'dqn'
+Runner.create_environment_fn = @gym_lib.create_gym_environment
+Runner.num_iterations = 500
+Runner.training_steps = 1000
+Runner.evaluation_steps = 1000
+Runner.max_steps_per_episode = 500
+
+WrappedReplayBuffer.replay_capacity = 50000
+WrappedReplayBuffer.batch_size = 128
diff --git a/dopamine/agents/dqn/configs/dqn_cartpole.gin b/dopamine/agents/dqn/configs/dqn_cartpole.gin
new file mode 100644
index 00000000..9a33508e
--- /dev/null
+++ b/dopamine/agents/dqn/configs/dqn_cartpole.gin
@@ -0,0 +1,35 @@
+# Hyperparameters for a simple DQN-style Cartpole agent. The hyperparameters
+# chosen achieve reasonable performance.
+import dopamine.discrete_domains.gym_lib
+import dopamine.discrete_domains.run_experiment
+import dopamine.agents.dqn.dqn_agent
+import dopamine.replay_memory.circular_replay_buffer
+import gin.tf.external_configurables
+
+DQNAgent.observation_shape = %gym_lib.CARTPOLE_OBSERVATION_SHAPE
+DQNAgent.observation_dtype = %gym_lib.CARTPOLE_OBSERVATION_DTYPE
+DQNAgent.stack_size = %gym_lib.CARTPOLE_STACK_SIZE
+DQNAgent.network = @gym_lib.cartpole_dqn_network
+DQNAgent.gamma = 0.99
+DQNAgent.update_horizon = 1
+DQNAgent.min_replay_history = 500
+DQNAgent.update_period = 4
+DQNAgent.target_update_period = 100
+DQNAgent.epsilon_fn = @dqn_agent.identity_epsilon
+DQNAgent.tf_device = '/gpu:0' # use '/cpu:*' for non-GPU version
+DQNAgent.optimizer = @tf.train.AdamOptimizer()
+
+tf.train.AdamOptimizer.learning_rate = 0.001
+tf.train.AdamOptimizer.epsilon = 0.0003125
+
+create_gym_environment.environment_name = 'CartPole'
+create_gym_environment.version = 'v0'
+create_agent.agent_name = 'dqn'
+Runner.create_environment_fn = @gym_lib.create_gym_environment
+Runner.num_iterations = 500
+Runner.training_steps = 1000
+Runner.evaluation_steps = 1000
+Runner.max_steps_per_episode = 200 # Default max episode length.
+
+WrappedReplayBuffer.replay_capacity = 50000
+WrappedReplayBuffer.batch_size = 128
diff --git a/dopamine/agents/dqn/configs/dqn_icml.gin b/dopamine/agents/dqn/configs/dqn_icml.gin
index c4dcb242..eea97962 100644
--- a/dopamine/agents/dqn/configs/dqn_icml.gin
+++ b/dopamine/agents/dqn/configs/dqn_icml.gin
@@ -1,5 +1,6 @@
# Hyperparameters used for reporting DQN results in Bellemare et al. (2017).
-import dopamine.atari.run_experiment
+import dopamine.discrete_domains.atari_lib
+import dopamine.discrete_domains.run_experiment
import dopamine.agents.dqn.dqn_agent
import dopamine.replay_memory.circular_replay_buffer
import gin.tf.external_configurables
@@ -21,13 +22,16 @@ tf.train.RMSPropOptimizer.momentum = 0.0
tf.train.RMSPropOptimizer.epsilon = 0.00001
tf.train.RMSPropOptimizer.centered = True
-Runner.game_name = 'Pong'
+atari_lib.create_atari_environment.game_name = 'Pong'
# Deterministic ALE version used in the DQN Nature paper (Mnih et al., 2015).
-Runner.sticky_actions = False
+atari_lib.create_atari_environment.sticky_actions = False
+create_agent.agent_name = 'dqn'
Runner.num_iterations = 200
Runner.training_steps = 250000 # agent steps
Runner.evaluation_steps = 125000 # agent steps
Runner.max_steps_per_episode = 27000 # agent steps
+AtariPreprocessing.terminal_on_life_loss = True
+
WrappedReplayBuffer.replay_capacity = 1000000
WrappedReplayBuffer.batch_size = 32
diff --git a/dopamine/agents/dqn/configs/dqn_nature.gin b/dopamine/agents/dqn/configs/dqn_nature.gin
index 024bff4c..4cf4664f 100644
--- a/dopamine/agents/dqn/configs/dqn_nature.gin
+++ b/dopamine/agents/dqn/configs/dqn_nature.gin
@@ -1,6 +1,6 @@
# Hyperparameters used in Mnih et al. (2015).
-import dopamine.atari.preprocessing
-import dopamine.atari.run_experiment
+import dopamine.discrete_domains.atari_lib
+import dopamine.discrete_domains.run_experiment
import dopamine.agents.dqn.dqn_agent
import dopamine.replay_memory.circular_replay_buffer
import gin.tf.external_configurables
@@ -22,9 +22,10 @@ tf.train.RMSPropOptimizer.momentum = 0.0
tf.train.RMSPropOptimizer.epsilon = 0.00001
tf.train.RMSPropOptimizer.centered = True
-Runner.game_name = 'Pong'
+atari_lib.create_atari_environment.game_name = 'Pong'
# Deterministic ALE version used in the DQN Nature paper (Mnih et al., 2015).
-Runner.sticky_actions = False
+atari_lib.create_atari_environment.sticky_actions = False
+create_agent.agent_name = 'dqn'
Runner.num_iterations = 200
Runner.training_steps = 250000 # agent steps
Runner.evaluation_steps = 125000 # agent steps
diff --git a/dopamine/agents/dqn/dqn_agent.py b/dopamine/agents/dqn/dqn_agent.py
index 35094415..6d693444 100644
--- a/dopamine/agents/dqn/dqn_agent.py
+++ b/dopamine/agents/dqn/dqn_agent.py
@@ -25,7 +25,9 @@
+from dopamine.discrete_domains import atari_lib
from dopamine.replay_memory import circular_replay_buffer
+from dopamine.common import get_checkpoint_duration
import numpy as np
import tensorflow as tf
@@ -34,11 +36,14 @@
slim = tf.contrib.slim
-NATURE_DQN_OBSERVATION_SHAPE = (84, 84) # Size of downscaled Atari 2600 frame.
-NATURE_DQN_DTYPE = tf.uint8 # DType of Atari 2600 observations.
-NATURE_DQN_STACK_SIZE = 4 # Number of frames in the state stack.
+# These are aliases which are used by other classes.
+NATURE_DQN_OBSERVATION_SHAPE = atari_lib.NATURE_DQN_OBSERVATION_SHAPE
+NATURE_DQN_DTYPE = atari_lib.NATURE_DQN_DTYPE
+NATURE_DQN_STACK_SIZE = atari_lib.NATURE_DQN_STACK_SIZE
+nature_dqn_network = atari_lib.nature_dqn_network
+@gin.configurable
def linearly_decaying_epsilon(decay_period, step, warmup_steps, epsilon):
"""Returns the current epsilon for the agent's epsilon-greedy policy.
@@ -63,6 +68,12 @@ def linearly_decaying_epsilon(decay_period, step, warmup_steps, epsilon):
return epsilon + bonus
+@gin.configurable
+def identity_epsilon(unused_decay_period, unused_step, unused_warmup_steps,
+ epsilon):
+ return epsilon
+
+
@gin.configurable
class DQNAgent(object):
"""An implementation of the DQN agent."""
@@ -70,9 +81,10 @@ class DQNAgent(object):
def __init__(self,
sess,
num_actions,
- observation_shape=NATURE_DQN_OBSERVATION_SHAPE,
- observation_dtype=NATURE_DQN_DTYPE,
- stack_size=NATURE_DQN_STACK_SIZE,
+ observation_shape=atari_lib.NATURE_DQN_OBSERVATION_SHAPE,
+ observation_dtype=atari_lib.NATURE_DQN_DTYPE,
+ stack_size=atari_lib.NATURE_DQN_STACK_SIZE,
+ network=atari_lib.nature_dqn_network,
gamma=0.99,
update_horizon=1,
min_replay_history=20000,
@@ -84,7 +96,7 @@ def __init__(self,
epsilon_decay_period=250000,
tf_device='/cpu:*',
use_staging=True,
- max_tf_checkpoints_to_keep=3,
+ max_tf_checkpoints_to_keep=get_checkpoint_duration(),
optimizer=tf.train.RMSPropOptimizer(
learning_rate=0.00025,
decay=0.95,
@@ -102,6 +114,11 @@ def __init__(self,
observation_dtype: tf.DType, specifies the type of the observations. Note
that if your inputs are continuous, you should set this to tf.float32.
stack_size: int, number of frames to use in state stack.
+ network: function expecting three parameters:
+ (num_actions, network_type, state). This function will return the
+ network_type object containing the tensors output by the network.
+ See dopamine.discrete_domains.atari_lib.nature_dqn_network as
+ an example.
gamma: float, discount factor with the usual RL meaning.
update_horizon: int, horizon at which updates are performed, the 'n' in
n-step update.
@@ -146,6 +163,7 @@ def __init__(self,
self.observation_shape = tuple(observation_shape)
self.observation_dtype = observation_dtype
self.stack_size = stack_size
+ self.network = network
self.gamma = gamma
self.update_horizon = update_horizon
self.cumulative_gamma = math.pow(gamma, update_horizon)
@@ -165,10 +183,7 @@ def __init__(self,
with tf.device(tf_device):
# Create a placeholder for the state input to the DQN network.
# The last axis indicates the number of consecutive frames stacked.
- state_shape = (1,) + self.observation_shape + (stack_size,)
- self.state = np.zeros(state_shape)
- self.state_ph = tf.placeholder(self.observation_dtype, state_shape,
- name='state_ph')
+ self._init_placeholder()
self._replay = self._build_replay_buffer(use_staging)
self._build_networks()
@@ -187,6 +202,12 @@ def __init__(self,
self._observation = None
self._last_observation = None
+ def _init_placeholder(self):
+ _state_shape = (1,) + self.observation_shape + (self.stack_size,)
+ self.state = np.zeros(_state_shape)
+ self.state_ph = tf.placeholder(self.observation_dtype, _state_shape,
+ name='state_ph')
+
def _get_network_type(self):
"""Returns the type of the outputs of a Q value network.
@@ -204,15 +225,7 @@ def _network_template(self, state):
Returns:
net: _network_type object containing the tensors output by the network.
"""
- net = tf.cast(state, tf.float32)
- net = tf.div(net, 255.)
- net = slim.conv2d(net, 32, [8, 8], stride=4)
- net = slim.conv2d(net, 64, [4, 4], stride=2)
- net = slim.conv2d(net, 64, [3, 3], stride=1)
- net = slim.flatten(net)
- net = slim.fully_connected(net, 512)
- q_values = slim.fully_connected(net, self.num_actions, activation_fn=None)
- return self._get_network_type()(q_values)
+ return self.network(self.num_actions, self._get_network_type(), state)
def _build_networks(self):
"""Builds the Q-value network computations needed for acting and training.
@@ -378,11 +391,14 @@ def _select_action(self):
Returns:
int, the selected action.
"""
- epsilon = self.epsilon_eval if self.eval_mode else self.epsilon_fn(
- self.epsilon_decay_period,
- self.training_steps,
- self.min_replay_history,
- self.epsilon_train)
+ if self.eval_mode:
+ epsilon = self.epsilon_eval
+ else:
+ epsilon = self.epsilon_fn(
+ self.epsilon_decay_period,
+ self.training_steps,
+ self.min_replay_history,
+ self.epsilon_train)
if random.random() <= epsilon:
# Choose a random action with probability epsilon.
return random.randint(0, self.num_actions - 1)
@@ -427,8 +443,6 @@ def _record_observation(self, observation):
"""
# Set current observation. We do the reshaping to handle environments
# without frame stacking.
- observation = np.reshape(observation, self.observation_shape)
- self._observation = observation[..., 0]
self._observation = np.reshape(observation, self.observation_shape)
# Swap out the oldest frame with the current frame.
self.state = np.roll(self.state, -1, axis=-1)
diff --git a/dopamine/agents/implicit_quantile/configs/implicit_quantile.gin b/dopamine/agents/implicit_quantile/configs/implicit_quantile.gin
index 584748fb..76b37f75 100644
--- a/dopamine/agents/implicit_quantile/configs/implicit_quantile.gin
+++ b/dopamine/agents/implicit_quantile/configs/implicit_quantile.gin
@@ -1,10 +1,10 @@
# Hyperparameters follow Dabney et al. (2018), but we modify as necessary to
# match those used in Rainbow (Hessel et al., 2018), to ensure apples-to-apples
# comparison.
-
import dopamine.agents.implicit_quantile.implicit_quantile_agent
import dopamine.agents.rainbow.rainbow_agent
-import dopamine.atari.run_experiment
+import dopamine.discrete_domains.atari_lib
+import dopamine.discrete_domains.run_experiment
import dopamine.replay_memory.prioritized_replay_buffer
import gin.tf.external_configurables
@@ -25,12 +25,13 @@ RainbowAgent.replay_scheme = 'uniform'
RainbowAgent.tf_device = '/gpu:0' # '/cpu:*' use for non-GPU version
RainbowAgent.optimizer = @tf.train.AdamOptimizer()
-tf.train.AdamOptimizer.learning_rate = 0.0000625
-tf.train.AdamOptimizer.epsilon = 0.00015
+tf.train.AdamOptimizer.learning_rate = 0.00005
+tf.train.AdamOptimizer.epsilon = 0.0003125
-Runner.game_name = 'Pong'
+atari_lib.create_atari_environment.game_name = 'Pong'
# Sticky actions with probability 0.25, as suggested by (Machado et al., 2017).
-Runner.sticky_actions = True
+atari_lib.create_atari_environment.sticky_actions = True
+create_agent.agent_name = 'implicit_quantile'
Runner.num_iterations = 200
Runner.training_steps = 250000
Runner.evaluation_steps = 125000
diff --git a/dopamine/agents/implicit_quantile/configs/implicit_quantile_icml.gin b/dopamine/agents/implicit_quantile/configs/implicit_quantile_icml.gin
index 265dde13..61761f6c 100644
--- a/dopamine/agents/implicit_quantile/configs/implicit_quantile_icml.gin
+++ b/dopamine/agents/implicit_quantile/configs/implicit_quantile_icml.gin
@@ -1,7 +1,8 @@
-# Hyperparameters follow Dabney et al. (2018)
+# Hyperparameters follow Dabney et al. (2018).
import dopamine.agents.implicit_quantile.implicit_quantile_agent
import dopamine.agents.rainbow.rainbow_agent
-import dopamine.atari.run_experiment
+import dopamine.discrete_domains.atari_lib
+import dopamine.discrete_domains.run_experiment
import dopamine.replay_memory.prioritized_replay_buffer
import gin.tf.external_configurables
@@ -24,8 +25,9 @@ RainbowAgent.optimizer = @tf.train.AdamOptimizer()
tf.train.AdamOptimizer.learning_rate = 0.00005
tf.train.AdamOptimizer.epsilon = 0.0003125
-Runner.game_name = 'Pong'
-Runner.sticky_actions = False
+atari_lib.create_atari_environment.game_name = 'Pong'
+atari_lib.create_atari_environment.sticky_actions = False
+create_agent.agent_name = 'implicit_quantile'
Runner.num_iterations = 200
Runner.training_steps = 250000
Runner.evaluation_steps = 125000
diff --git a/dopamine/agents/implicit_quantile/implicit_quantile_agent.py b/dopamine/agents/implicit_quantile/implicit_quantile_agent.py
index 1b689bce..f0285d77 100644
--- a/dopamine/agents/implicit_quantile/implicit_quantile_agent.py
+++ b/dopamine/agents/implicit_quantile/implicit_quantile_agent.py
@@ -24,11 +24,10 @@
import collections
-import math
from dopamine.agents.rainbow import rainbow_agent
-import numpy as np
+from dopamine.discrete_domains import atari_lib
import tensorflow as tf
import gin.tf
@@ -43,6 +42,7 @@ class ImplicitQuantileAgent(rainbow_agent.RainbowAgent):
def __init__(self,
sess,
num_actions,
+ network=atari_lib.implicit_quantile_network,
kappa=1.0,
num_tau_samples=32,
num_tau_prime_samples=32,
@@ -59,6 +59,11 @@ def __init__(self,
Args:
sess: `tf.Session` object for running associated ops.
num_actions: int, number of actions the agent can take at any state.
+ network: function expecting three parameters:
+ (num_actions, network_type, state). This function will return the
+ network_type object containing the tensors output by the network.
+ See dopamine.discrete_domains.atari_lib.nature_dqn_network as
+ an example.
kappa: float, Huber loss cutoff.
num_tau_samples: int, number of online quantile samples for loss
estimation.
@@ -89,6 +94,7 @@ def __init__(self,
super(ImplicitQuantileAgent, self).__init__(
sess=sess,
num_actions=num_actions,
+ network=network,
summary_writer=summary_writer,
summary_writing_frequency=summary_writing_frequency)
@@ -113,50 +119,8 @@ def _network_template(self, state, num_quantiles):
Returns:
_network_type object containing quantile value outputs of the network.
"""
-
- weights_initializer = slim.variance_scaling_initializer(
- factor=1.0 / np.sqrt(3.0), mode='FAN_IN', uniform=True)
-
- state_net = tf.cast(state, tf.float32)
- state_net = tf.div(state_net, 255.)
- state_net = slim.conv2d(
- state_net, 32, [8, 8], stride=4,
- weights_initializer=weights_initializer)
- state_net = slim.conv2d(
- state_net, 64, [4, 4], stride=2,
- weights_initializer=weights_initializer)
- state_net = slim.conv2d(
- state_net, 64, [3, 3], stride=1,
- weights_initializer=weights_initializer)
- state_net = slim.flatten(state_net)
- state_net_size = state_net.get_shape().as_list()[-1]
- state_net_tiled = tf.tile(state_net, [num_quantiles, 1])
-
- batch_size = state_net.get_shape().as_list()[0]
- quantiles_shape = [num_quantiles * batch_size, 1]
- quantiles = tf.random_uniform(
- quantiles_shape, minval=0, maxval=1, dtype=tf.float32)
-
- quantile_net = tf.tile(quantiles, [1, self.quantile_embedding_dim])
- pi = tf.constant(math.pi)
- quantile_net = tf.cast(tf.range(
- 1, self.quantile_embedding_dim + 1, 1), tf.float32) * pi * quantile_net
- quantile_net = tf.cos(quantile_net)
- quantile_net = slim.fully_connected(quantile_net, state_net_size,
- weights_initializer=weights_initializer)
- # Hadamard product.
- net = tf.multiply(state_net_tiled, quantile_net)
-
- net = slim.fully_connected(
- net, 512, weights_initializer=weights_initializer)
- quantile_values = slim.fully_connected(
- net,
- self.num_actions,
- activation_fn=None,
- weights_initializer=weights_initializer)
-
- return self._get_network_type()(quantile_values=quantile_values,
- quantiles=quantiles)
+ return self.network(self.num_actions, self.quantile_embedding_dim,
+ self._get_network_type(), state, num_quantiles)
def _build_networks(self):
"""Builds the IQN computations needed for acting and training.
diff --git a/dopamine/agents/rainbow/configs/c51.gin b/dopamine/agents/rainbow/configs/c51.gin
index a573b893..4374e826 100644
--- a/dopamine/agents/rainbow/configs/c51.gin
+++ b/dopamine/agents/rainbow/configs/c51.gin
@@ -2,7 +2,8 @@
# modify as necessary to match those used in Rainbow (Hessel et al., 2018), to
# ensure apples-to-apples comparison.
import dopamine.agents.rainbow.rainbow_agent
-import dopamine.atari.run_experiment
+import dopamine.discrete_domains.atari_lib
+import dopamine.discrete_domains.run_experiment
import dopamine.replay_memory.prioritized_replay_buffer
import gin.tf.external_configurables
@@ -23,9 +24,10 @@ RainbowAgent.optimizer = @tf.train.AdamOptimizer()
tf.train.AdamOptimizer.learning_rate = 0.00025
tf.train.AdamOptimizer.epsilon = 0.0003125
-Runner.game_name = 'Pong'
+atari_lib.create_atari_environment.game_name = 'Pong'
# Sticky actions with probability 0.25, as suggested by (Machado et al., 2017).
-Runner.sticky_actions = True
+atari_lib.create_atari_environment.sticky_actions = True
+create_agent.agent_name = 'rainbow'
Runner.num_iterations = 200
Runner.training_steps = 250000 # agent steps
Runner.evaluation_steps = 125000 # agent steps
diff --git a/dopamine/agents/rainbow/configs/c51_acrobot.gin b/dopamine/agents/rainbow/configs/c51_acrobot.gin
new file mode 100644
index 00000000..9acd3b50
--- /dev/null
+++ b/dopamine/agents/rainbow/configs/c51_acrobot.gin
@@ -0,0 +1,39 @@
+# Hyperparameters for a simple C51-style Acrobot agent. The hyperparameters
+# chosen achieve reasonable performance.
+import dopamine.agents.dqn.dqn_agent
+import dopamine.agents.rainbow.rainbow_agent
+import dopamine.discrete_domains.gym_lib
+import dopamine.discrete_domains.run_experiment
+import dopamine.replay_memory.prioritized_replay_buffer
+import gin.tf.external_configurables
+
+RainbowAgent.observation_shape = %gym_lib.ACROBOT_OBSERVATION_SHAPE
+RainbowAgent.observation_dtype = %gym_lib.ACROBOT_OBSERVATION_DTYPE
+RainbowAgent.stack_size = %gym_lib.ACROBOT_STACK_SIZE
+RainbowAgent.network = @gym_lib.acrobot_rainbow_network
+RainbowAgent.num_atoms = 51
+RainbowAgent.vmax = 10.
+RainbowAgent.gamma = 0.99
+RainbowAgent.update_horizon = 1
+RainbowAgent.min_replay_history = 500
+RainbowAgent.update_period = 4
+RainbowAgent.target_update_period = 100
+RainbowAgent.epsilon_fn = @dqn_agent.identity_epsilon
+RainbowAgent.replay_scheme = 'uniform'
+RainbowAgent.tf_device = '/gpu:0' # use '/cpu:*' for non-GPU version
+RainbowAgent.optimizer = @tf.train.AdamOptimizer()
+
+tf.train.AdamOptimizer.learning_rate = 0.1
+tf.train.AdamOptimizer.epsilon = 0.0003125
+
+create_gym_environment.environment_name = 'Acrobot'
+create_gym_environment.version = 'v1'
+create_agent.agent_name = 'rainbow'
+Runner.create_environment_fn = @gym_lib.create_gym_environment
+Runner.num_iterations = 500
+Runner.training_steps = 1000
+Runner.evaluation_steps = 1000
+Runner.max_steps_per_episode = 500
+
+WrappedPrioritizedReplayBuffer.replay_capacity = 50000
+WrappedPrioritizedReplayBuffer.batch_size = 128
diff --git a/dopamine/agents/rainbow/configs/c51_cartpole.gin b/dopamine/agents/rainbow/configs/c51_cartpole.gin
new file mode 100644
index 00000000..16390e27
--- /dev/null
+++ b/dopamine/agents/rainbow/configs/c51_cartpole.gin
@@ -0,0 +1,39 @@
+# Hyperparameters for a simple C51-style Cartpole agent. The hyperparameters
+# chosen achieve reasonable performance.
+import dopamine.agents.dqn.dqn_agent
+import dopamine.agents.rainbow.rainbow_agent
+import dopamine.discrete_domains.gym_lib
+import dopamine.discrete_domains.run_experiment
+import dopamine.replay_memory.prioritized_replay_buffer
+import gin.tf.external_configurables
+
+RainbowAgent.observation_shape = %gym_lib.CARTPOLE_OBSERVATION_SHAPE
+RainbowAgent.observation_dtype = %gym_lib.CARTPOLE_OBSERVATION_DTYPE
+RainbowAgent.stack_size = %gym_lib.CARTPOLE_STACK_SIZE
+RainbowAgent.network = @gym_lib.cartpole_rainbow_network
+RainbowAgent.num_atoms = 51
+RainbowAgent.vmax = 10.
+RainbowAgent.gamma = 0.99
+RainbowAgent.update_horizon = 1
+RainbowAgent.min_replay_history = 500
+RainbowAgent.update_period = 4
+RainbowAgent.target_update_period = 100
+RainbowAgent.epsilon_fn = @dqn_agent.identity_epsilon
+RainbowAgent.replay_scheme = 'uniform'
+RainbowAgent.tf_device = '/gpu:0' # use '/cpu:*' for non-GPU version
+RainbowAgent.optimizer = @tf.train.AdamOptimizer()
+
+tf.train.AdamOptimizer.learning_rate = 0.001
+tf.train.AdamOptimizer.epsilon = 0.0003125
+
+create_gym_environment.environment_name = 'CartPole'
+create_gym_environment.version = 'v0'
+create_agent.agent_name = 'rainbow'
+Runner.create_environment_fn = @gym_lib.create_gym_environment
+Runner.num_iterations = 500
+Runner.training_steps = 1000
+Runner.evaluation_steps = 1000
+Runner.max_steps_per_episode = 200 # Default max episode length.
+
+WrappedPrioritizedReplayBuffer.replay_capacity = 50000
+WrappedPrioritizedReplayBuffer.batch_size = 128
diff --git a/dopamine/agents/rainbow/configs/c51_icml.gin b/dopamine/agents/rainbow/configs/c51_icml.gin
index b06aa7d0..e148591c 100644
--- a/dopamine/agents/rainbow/configs/c51_icml.gin
+++ b/dopamine/agents/rainbow/configs/c51_icml.gin
@@ -1,7 +1,7 @@
# Hyperparameters used in Bellemare et al. (2017).
-import dopamine.atari.preprocessing
import dopamine.agents.rainbow.rainbow_agent
-import dopamine.atari.run_experiment
+import dopamine.discrete_domains.atari_lib
+import dopamine.discrete_domains.run_experiment
import dopamine.replay_memory.prioritized_replay_buffer
import gin.tf.external_configurables
@@ -22,9 +22,10 @@ RainbowAgent.optimizer = @tf.train.AdamOptimizer()
tf.train.AdamOptimizer.learning_rate = 0.00025
tf.train.AdamOptimizer.epsilon = 0.0003125
-Runner.game_name = 'Pong'
+atari_lib.create_atari_environment.game_name = 'Pong'
# Deterministic ALE version used in the DQN Nature paper (Mnih et al., 2015).
-Runner.sticky_actions = False
+atari_lib.create_atari_environment.sticky_actions = False
+create_agent.agent_name = 'rainbow'
Runner.num_iterations = 200
Runner.training_steps = 250000 # agent steps
Runner.evaluation_steps = 125000 # agent steps
diff --git a/dopamine/agents/rainbow/configs/rainbow.gin b/dopamine/agents/rainbow/configs/rainbow.gin
index 00046d2c..73fe1348 100644
--- a/dopamine/agents/rainbow/configs/rainbow.gin
+++ b/dopamine/agents/rainbow/configs/rainbow.gin
@@ -1,7 +1,8 @@
# Hyperparameters follow Hessel et al. (2018), except for sticky_actions,
# which was False (not using sticky actions) in the original paper.
import dopamine.agents.rainbow.rainbow_agent
-import dopamine.atari.run_experiment
+import dopamine.discrete_domains.atari_lib
+import dopamine.discrete_domains.run_experiment
import dopamine.replay_memory.prioritized_replay_buffer
import gin.tf.external_configurables
@@ -23,9 +24,10 @@ RainbowAgent.optimizer = @tf.train.AdamOptimizer()
tf.train.AdamOptimizer.learning_rate = 0.0000625
tf.train.AdamOptimizer.epsilon = 0.00015
-Runner.game_name = 'Pong'
+atari_lib.create_atari_environment.game_name = 'Pong'
# Sticky actions with probability 0.25, as suggested by (Machado et al., 2017).
-Runner.sticky_actions = True
+atari_lib.create_atari_environment.sticky_actions = True
+create_agent.agent_name = 'rainbow'
Runner.num_iterations = 200
Runner.training_steps = 250000 # agent steps
Runner.evaluation_steps = 125000 # agent steps
diff --git a/dopamine/agents/rainbow/configs/rainbow_aaai.gin b/dopamine/agents/rainbow/configs/rainbow_aaai.gin
index 48be0f6c..60ea72db 100644
--- a/dopamine/agents/rainbow/configs/rainbow_aaai.gin
+++ b/dopamine/agents/rainbow/configs/rainbow_aaai.gin
@@ -1,7 +1,7 @@
# Hyperparameters follow Hessel et al. (2018).
-import dopamine.atari.preprocessing
import dopamine.agents.rainbow.rainbow_agent
-import dopamine.atari.run_experiment
+import dopamine.discrete_domains.atari_lib
+import dopamine.discrete_domains.run_experiment
import dopamine.replay_memory.prioritized_replay_buffer
import gin.tf.external_configurables
@@ -23,9 +23,10 @@ RainbowAgent.optimizer = @tf.train.AdamOptimizer()
tf.train.AdamOptimizer.learning_rate = 0.0000625
tf.train.AdamOptimizer.epsilon = 0.00015
-Runner.game_name = 'Pong'
+atari_lib.create_atari_environment.game_name = 'Pong'
# Deterministic ALE version used in the AAAI paper.
-Runner.sticky_actions = False
+atari_lib.create_atari_environment.sticky_actions = False
+create_agent.agent_name = 'rainbow'
Runner.num_iterations = 200
Runner.training_steps = 250000 # agent steps
Runner.evaluation_steps = 125000 # agent steps
diff --git a/dopamine/agents/rainbow/configs/rainbow_acrobot.gin b/dopamine/agents/rainbow/configs/rainbow_acrobot.gin
new file mode 100644
index 00000000..e58ef8bf
--- /dev/null
+++ b/dopamine/agents/rainbow/configs/rainbow_acrobot.gin
@@ -0,0 +1,38 @@
+# Hyperparameters for a simple Rainbow-style Acrobot agent. The hyperparameters
+# chosen achieve reasonable performance.
+import dopamine.agents.rainbow.rainbow_agent
+import dopamine.discrete_domains.gym_lib
+import dopamine.discrete_domains.run_experiment
+import dopamine.replay_memory.prioritized_replay_buffer
+import gin.tf.external_configurables
+
+RainbowAgent.observation_shape = %gym_lib.ACROBOT_OBSERVATION_SHAPE
+RainbowAgent.observation_dtype = %gym_lib.ACROBOT_OBSERVATION_DTYPE
+RainbowAgent.stack_size = %gym_lib.ACROBOT_STACK_SIZE
+RainbowAgent.network = @gym_lib.acrobot_rainbow_network
+RainbowAgent.num_atoms = 51
+RainbowAgent.vmax = 10.
+RainbowAgent.gamma = 0.99
+RainbowAgent.update_horizon = 3
+RainbowAgent.min_replay_history = 500
+RainbowAgent.update_period = 4
+RainbowAgent.target_update_period = 100
+RainbowAgent.epsilon_fn = @dqn_agent.identity_epsilon
+RainbowAgent.replay_scheme = 'prioritized'
+RainbowAgent.tf_device = '/gpu:0' # use '/cpu:*' for non-GPU version
+RainbowAgent.optimizer = @tf.train.AdamOptimizer()
+
+tf.train.AdamOptimizer.learning_rate = 0.09
+tf.train.AdamOptimizer.epsilon = 0.0003125
+
+create_gym_environment.environment_name = 'Acrobot'
+create_gym_environment.version = 'v1'
+create_agent.agent_name = 'rainbow'
+Runner.create_environment_fn = @gym_lib.create_gym_environment
+Runner.num_iterations = 500
+Runner.training_steps = 1000
+Runner.evaluation_steps = 1000
+Runner.max_steps_per_episode = 500
+
+WrappedPrioritizedReplayBuffer.replay_capacity = 50000
+WrappedPrioritizedReplayBuffer.batch_size = 128
diff --git a/dopamine/agents/rainbow/configs/rainbow_cartpole.gin b/dopamine/agents/rainbow/configs/rainbow_cartpole.gin
new file mode 100644
index 00000000..3f4c3dc7
--- /dev/null
+++ b/dopamine/agents/rainbow/configs/rainbow_cartpole.gin
@@ -0,0 +1,39 @@
+# Hyperparameters for a simple Rainbow-style Cartpole agent. The
+# hyperparameters chosen achieve reasonable performance.
+import dopamine.agents.dqn.dqn_agent
+import dopamine.agents.rainbow.rainbow_agent
+import dopamine.discrete_domains.gym_lib
+import dopamine.discrete_domains.run_experiment
+import dopamine.replay_memory.prioritized_replay_buffer
+import gin.tf.external_configurables
+
+RainbowAgent.observation_shape = %gym_lib.CARTPOLE_OBSERVATION_SHAPE
+RainbowAgent.observation_dtype = %gym_lib.CARTPOLE_OBSERVATION_DTYPE
+RainbowAgent.stack_size = %gym_lib.CARTPOLE_STACK_SIZE
+RainbowAgent.network = @gym_lib.cartpole_rainbow_network
+RainbowAgent.num_atoms = 51
+RainbowAgent.vmax = 10.
+RainbowAgent.gamma = 0.99
+RainbowAgent.update_horizon = 3
+RainbowAgent.min_replay_history = 500
+RainbowAgent.update_period = 4
+RainbowAgent.target_update_period = 100
+RainbowAgent.epsilon_fn = @dqn_agent.identity_epsilon
+RainbowAgent.replay_scheme = 'prioritized'
+RainbowAgent.tf_device = '/gpu:0' # use '/cpu:*' for non-GPU version
+RainbowAgent.optimizer = @tf.train.AdamOptimizer()
+
+tf.train.AdamOptimizer.learning_rate = 0.09
+tf.train.AdamOptimizer.epsilon = 0.0003125
+
+create_gym_environment.environment_name = 'CartPole'
+create_gym_environment.version = 'v0'
+create_agent.agent_name = 'rainbow'
+Runner.create_environment_fn = @gym_lib.create_gym_environment
+Runner.num_iterations = 500
+Runner.training_steps = 1000
+Runner.evaluation_steps = 1000
+Runner.max_steps_per_episode = 200 # Default max episode length.
+
+WrappedPrioritizedReplayBuffer.replay_capacity = 50000
+WrappedPrioritizedReplayBuffer.batch_size = 128
diff --git a/dopamine/agents/rainbow/rainbow_agent.py b/dopamine/agents/rainbow/rainbow_agent.py
index 67ec08a1..29aaddb1 100644
--- a/dopamine/agents/rainbow/rainbow_agent.py
+++ b/dopamine/agents/rainbow/rainbow_agent.py
@@ -42,8 +42,8 @@
from dopamine.agents.dqn import dqn_agent
+from dopamine.discrete_domains import atari_lib
from dopamine.replay_memory import prioritized_replay_buffer
-import numpy as np
import tensorflow as tf
import gin.tf
@@ -61,6 +61,7 @@ def __init__(self,
observation_shape=dqn_agent.NATURE_DQN_OBSERVATION_SHAPE,
observation_dtype=dqn_agent.NATURE_DQN_DTYPE,
stack_size=dqn_agent.NATURE_DQN_STACK_SIZE,
+ network=atari_lib.rainbow_network,
num_atoms=51,
vmax=10.,
gamma=0.99,
@@ -89,6 +90,11 @@ def __init__(self,
observation_dtype: tf.DType, specifies the type of the observations. Note
that if your inputs are continuous, you should set this to tf.float32.
stack_size: int, number of frames to use in state stack.
+ network: function expecting three parameters:
+ (num_actions, network_type, state). This function will return the
+ network_type object containing the tensors output by the network.
+ See dopamine.discrete_domains.atari_lib.rainbow_network as
+ an example.
num_atoms: int, the number of buckets of the value function distribution.
vmax: float, the value distribution support is [-vmax, vmax].
gamma: float, discount factor with the usual RL meaning.
@@ -124,12 +130,14 @@ def __init__(self,
# TODO(b/110897128): Make agent optimizer attribute private.
self.optimizer = optimizer
- super(RainbowAgent, self).__init__(
+ dqn_agent.DQNAgent.__init__(
+ self,
sess=sess,
num_actions=num_actions,
observation_shape=observation_shape,
observation_dtype=observation_dtype,
stack_size=stack_size,
+ network=network,
gamma=gamma,
update_horizon=update_horizon,
min_replay_history=min_replay_history,
@@ -163,30 +171,8 @@ def _network_template(self, state):
Returns:
net: _network_type object containing the tensors output by the network.
"""
- weights_initializer = slim.variance_scaling_initializer(
- factor=1.0 / np.sqrt(3.0), mode='FAN_IN', uniform=True)
-
- net = tf.cast(state, tf.float32)
- net = tf.div(net, 255.)
- net = slim.conv2d(
- net, 32, [8, 8], stride=4, weights_initializer=weights_initializer)
- net = slim.conv2d(
- net, 64, [4, 4], stride=2, weights_initializer=weights_initializer)
- net = slim.conv2d(
- net, 64, [3, 3], stride=1, weights_initializer=weights_initializer)
- net = slim.flatten(net)
- net = slim.fully_connected(
- net, 512, weights_initializer=weights_initializer)
- net = slim.fully_connected(
- net,
- self.num_actions * self._num_atoms,
- activation_fn=None,
- weights_initializer=weights_initializer)
-
- logits = tf.reshape(net, [-1, self.num_actions, self._num_atoms])
- probabilities = tf.contrib.layers.softmax(logits)
- q_values = tf.reduce_sum(self._support * probabilities, axis=2)
- return self._get_network_type()(q_values, logits, probabilities)
+ return self.network(self.num_actions, self._num_atoms, self._support,
+ self._get_network_type(), state)
def _build_replay_buffer(self, use_staging):
"""Creates the replay buffer used by the agent.
@@ -340,8 +326,10 @@ def _store_transition(self,
maximum ever seen [Schaul et al., 2015].
"""
if priority is None:
- priority = (1. if self._replay_scheme == 'uniform' else
- self._replay.memory.sum_tree.max_recorded_priority)
+ if self._replay_scheme == 'uniform':
+ priority = 1.
+ else:
+ priority = self._replay.memory.sum_tree.max_recorded_priority
if not self.eval_mode:
self._replay.add(last_observation, action, reward, is_terminal, priority)
diff --git a/dopamine/atari/__init__.py b/dopamine/atari/__init__.py
deleted file mode 100644
index 920cbb5e..00000000
--- a/dopamine/atari/__init__.py
+++ /dev/null
@@ -1,15 +0,0 @@
-# coding=utf-8
-# Copyright 2018 The Dopamine Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
diff --git a/dopamine/atari/train.py b/dopamine/atari/train.py
deleted file mode 100644
index 548bbcbf..00000000
--- a/dopamine/atari/train.py
+++ /dev/null
@@ -1,146 +0,0 @@
-# coding=utf-8
-# Copyright 2018 The Dopamine Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-r"""The entry point for running an agent on an Atari 2600 domain.
-
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-
-from absl import app
-from absl import flags
-from dopamine.agents.dqn import dqn_agent
-from dopamine.agents.implicit_quantile import implicit_quantile_agent
-from dopamine.agents.rainbow import rainbow_agent
-from dopamine.atari import run_experiment
-
-import tensorflow as tf
-
-
-flags.DEFINE_bool('debug_mode', False,
- 'If set to true, the agent will output in-episode statistics '
- 'to Tensorboard. Disabled by default as this results in '
- 'slower training.')
-flags.DEFINE_string('agent_name', None,
- 'Name of the agent. Must be one of '
- '(dqn, rainbow, implicit_quantile)')
-flags.DEFINE_string('base_dir', None,
- 'Base directory to host all required sub-directories.')
-flags.DEFINE_multi_string(
- 'gin_files', [], 'List of paths to gin configuration files (e.g.'
- '"dopamine/agents/dqn/dqn.gin").')
-flags.DEFINE_multi_string(
- 'gin_bindings', [],
- 'Gin bindings to override the values set in the config files '
- '(e.g. "DQNAgent.epsilon_train=0.1",'
- ' "create_environment.game_name="Pong"").')
-flags.DEFINE_string(
- 'schedule', 'continuous_train_and_eval',
- 'The schedule with which to run the experiment and choose an appropriate '
- 'Runner. Supported choices are '
- '{continuous_train, continuous_train_and_eval}.')
-
-FLAGS = flags.FLAGS
-
-
-
-def create_agent(sess, environment, summary_writer=None):
- """Creates a DQN agent.
-
- Args:
- sess: A `tf.Session` object for running associated ops.
- environment: An Atari 2600 Gym environment.
- summary_writer: A Tensorflow summary writer to pass to the agent
- for in-agent training statistics in Tensorboard.
-
- Returns:
- agent: An RL agent.
-
- Raises:
- ValueError: If `agent_name` is not in supported list.
- """
- if not FLAGS.debug_mode:
- summary_writer = None
- if FLAGS.agent_name == 'dqn':
- return dqn_agent.DQNAgent(sess, num_actions=environment.action_space.n,
- summary_writer=summary_writer)
- elif FLAGS.agent_name == 'rainbow':
- return rainbow_agent.RainbowAgent(
- sess, num_actions=environment.action_space.n,
- summary_writer=summary_writer)
- elif FLAGS.agent_name == 'implicit_quantile':
- return implicit_quantile_agent.ImplicitQuantileAgent(
- sess, num_actions=environment.action_space.n,
- summary_writer=summary_writer)
- else:
- raise ValueError('Unknown agent: {}'.format(FLAGS.agent_name))
-
-
-def create_runner(base_dir, create_agent_fn):
- """Creates an experiment Runner.
-
- Args:
- base_dir: str, base directory for hosting all subdirectories.
- create_agent_fn: A function that takes as args a Tensorflow session and an
- Atari 2600 Gym environment, and returns an agent.
-
- Returns:
- runner: A `run_experiment.Runner` like object.
-
- Raises:
- ValueError: When an unknown schedule is encountered.
- """
- assert base_dir is not None
- # Continuously runs training and evaluation until max num_iterations is hit.
- if FLAGS.schedule == 'continuous_train_and_eval':
- return run_experiment.Runner(base_dir, create_agent_fn)
- # Continuously runs training until max num_iterations is hit.
- elif FLAGS.schedule == 'continuous_train':
- return run_experiment.TrainRunner(base_dir, create_agent_fn)
- else:
- raise ValueError('Unknown schedule: {}'.format(FLAGS.schedule))
-
-
-def launch_experiment(create_runner_fn, create_agent_fn):
- """Launches the experiment.
-
- Args:
- create_runner_fn: A function that takes as args a base directory and a
- function for creating an agent and returns a `Runner`-like object.
- create_agent_fn: A function that takes as args a Tensorflow session and an
- Atari 2600 Gym environment, and returns an agent.
- """
- run_experiment.load_gin_configs(FLAGS.gin_files, FLAGS.gin_bindings)
- runner = create_runner_fn(FLAGS.base_dir, create_agent_fn)
- runner.run_experiment()
-
-
-def main(unused_argv):
- """Main method.
-
- Args:
- unused_argv: Arguments (unused).
- """
- tf.logging.set_verbosity(tf.logging.INFO)
- launch_experiment(create_runner, create_agent)
-
-
-if __name__ == '__main__':
- flags.mark_flag_as_required('agent_name')
- flags.mark_flag_as_required('base_dir')
- app.run(main)
diff --git a/dopamine/colab/README.md b/dopamine/colab/README.md
index 3b822146..7f9bc0d1 100644
--- a/dopamine/colab/README.md
+++ b/dopamine/colab/README.md
@@ -24,3 +24,8 @@ we illustrate how to load and visualize the logs data produced by Dopamine.
In this
[colab](https://colab.research.google.com/github/google/dopamine/blob/master/dopamine/colab/tensorboard.ipynb)
we illustrate how to download and visualize different agents with Tensorboard.
+
+## Training on Cartpole
+In this
+[colab](https://colab.research.google.com/github/google/dopamine/blob/master/dopamine/colab/cartpole.ipynb)
+we illustrate how to train DQN and C51 on the Cartpole environment.
diff --git a/dopamine/colab/agents.ipynb b/dopamine/colab/agents.ipynb
index 7d4f6302..98f271fa 100644
--- a/dopamine/colab/agents.ipynb
+++ b/dopamine/colab/agents.ipynb
@@ -57,7 +57,8 @@
"# @title Install necessary packages.\n",
"!pip install --upgrade --no-cache-dir dopamine-rl\n",
"!pip install cmake\n",
- "!pip install atari_py"
+ "!pip install atari_py\n",
+ "!pip install gin-config"
],
"execution_count": 0,
"outputs": []
@@ -76,9 +77,10 @@
"import numpy as np\n",
"import os\n",
"from dopamine.agents.dqn import dqn_agent\n",
- "from dopamine.atari import run_experiment\n",
+ "from dopamine.discrete_domains import run_experiment\n",
"from dopamine.colab import utils as colab_utils\n",
"from absl import flags\n",
+ "import gin.tf\n",
"\n",
"BASE_PATH = '/tmp/colab_dope_run' # @param\n",
"GAME = 'Asterix' # @param"
@@ -120,11 +122,7 @@
"metadata": {
"id": "PUBRSmX6dfa3",
"colab_type": "code",
- "outputId": "f15fbc7b-a4bf-4480-ac2a-d43de77b84a8",
- "colab": {
- "base_uri": "https://localhost:8080/",
- "height": 359
- }
+ "colab": {}
},
"cell_type": "code",
"source": [
@@ -147,46 +145,24 @@
" \"\"\"The Runner class will expect a function of this type to create an agent.\"\"\"\n",
" return MyRandomDQNAgent(sess, num_actions=environment.action_space.n)\n",
"\n",
+ "random_dqn_config = \"\"\"\n",
+ "import dopamine.discrete_domains.atari_lib\n",
+ "import dopamine.discrete_domains.run_experiment\n",
+ "atari_lib.create_atari_environment.game_name = '{}'\n",
+ "atari_lib.create_atari_environment.sticky_actions = True\n",
+ "run_experiment.Runner.num_iterations = 200\n",
+ "run_experiment.Runner.training_steps = 10\n",
+ "run_experiment.Runner.max_steps_per_episode = 100\n",
+ "\"\"\".format(GAME)\n",
+ "gin.parse_config(random_dqn_config, skip_unknown=False)\n",
+ "\n",
"# Create the runner class with this agent. We use very small numbers of steps\n",
"# to terminate quickly, as this is mostly meant for demonstrating how one can\n",
- "# use the framework. We also explicitly terminate after 110 iterations (instead\n",
- "# of the standard 200) to demonstrate the plotting of partial runs.\n",
- "random_dqn_runner = run_experiment.Runner(LOG_PATH,\n",
- " create_random_dqn_agent,\n",
- " game_name=GAME,\n",
- " num_iterations=200,\n",
- " training_steps=10,\n",
- " evaluation_steps=10,\n",
- " max_steps_per_episode=100)"
+ "# use the framework.\n",
+ "random_dqn_runner = run_experiment.TrainRunner(LOG_PATH, create_random_dqn_agent)"
],
- "execution_count": 5,
- "outputs": [
- {
- "output_type": "stream",
- "text": [
- "INFO:tensorflow:Creating MyRandomDQNAgent agent with the following parameters:\n",
- "INFO:tensorflow:\t gamma: 0.990000\n",
- "INFO:tensorflow:\t update_horizon: 1.000000\n",
- "INFO:tensorflow:\t min_replay_history: 20000\n",
- "INFO:tensorflow:\t update_period: 4\n",
- "INFO:tensorflow:\t target_update_period: 8000\n",
- "INFO:tensorflow:\t epsilon_train: 0.010000\n",
- "INFO:tensorflow:\t epsilon_eval: 0.001000\n",
- "INFO:tensorflow:\t epsilon_decay_period: 250000\n",
- "INFO:tensorflow:\t tf_device: /cpu:*\n",
- "INFO:tensorflow:\t use_staging: True\n",
- "INFO:tensorflow:\t optimizer:
\n",
- "INFO:tensorflow:Creating a OutOfGraphReplayBuffer replay memory with the following parameters:\n",
- "INFO:tensorflow:\t observation_shape: 84\n",
- "INFO:tensorflow:\t stack_size: 4\n",
- "INFO:tensorflow:\t replay_capacity: 1000000\n",
- "INFO:tensorflow:\t batch_size: 32\n",
- "INFO:tensorflow:\t update_horizon: 1\n",
- "INFO:tensorflow:\t gamma: 0.990000\n"
- ],
- "name": "stdout"
- }
- ]
+ "execution_count": 0,
+ "outputs": []
},
{
"metadata": {
@@ -245,7 +221,7 @@
"plt.title(GAME)\n",
"plt.show()"
],
- "execution_count": 8,
+ "execution_count": 0,
"outputs": [
{
"output_type": "display_data",
@@ -322,17 +298,21 @@
" return StickyAgent(sess, num_actions=environment.action_space.n,\n",
" switch_prob=0.2)\n",
"\n",
+ "sticky_config = \"\"\"\n",
+ "import dopamine.discrete_domains.atari_lib\n",
+ "import dopamine.discrete_domains.run_experiment\n",
+ "atari_lib.create_atari_environment.game_name = '{}'\n",
+ "atari_lib.create_atari_environment.sticky_actions = True\n",
+ "run_experiment.Runner.num_iterations = 200\n",
+ "run_experiment.Runner.training_steps = 10\n",
+ "run_experiment.Runner.max_steps_per_episode = 100\n",
+ "\"\"\".format(GAME)\n",
+ "gin.parse_config(sticky_config, skip_unknown=False)\n",
+ "\n",
"# Create the runner class with this agent. We use very small numbers of steps\n",
"# to terminate quickly, as this is mostly meant for demonstrating how one can\n",
- "# use the framework. We also explicitly terminate after 110 iterations (instead\n",
- "# of the standard 200) to demonstrate the plotting of partial runs.\n",
- "sticky_runner = run_experiment.Runner(LOG_PATH,\n",
- " create_sticky_agent,\n",
- " game_name=GAME,\n",
- " num_iterations=200,\n",
- " training_steps=10,\n",
- " evaluation_steps=10,\n",
- " max_steps_per_episode=100)"
+ "# use the framework.\n",
+ "sticky_runner = run_experiment.TrainRunner(LOG_PATH, create_sticky_agent)"
],
"execution_count": 0,
"outputs": []
@@ -394,7 +374,7 @@
"plt.title(GAME)\n",
"plt.show()"
],
- "execution_count": 14,
+ "execution_count": 0,
"outputs": [
{
"output_type": "display_data",
diff --git a/dopamine/colab/cartpole.ipynb b/dopamine/colab/cartpole.ipynb
new file mode 100644
index 00000000..a9534d59
--- /dev/null
+++ b/dopamine/colab/cartpole.ipynb
@@ -0,0 +1,342 @@
+{
+ "nbformat": 4,
+ "nbformat_minor": 0,
+ "metadata": {
+ "colab": {
+ "name": "cartpole.ipynb",
+ "version": "0.3.2",
+ "provenance": [],
+ "toc_visible": true
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ }
+ },
+ "cells": [
+ {
+ "metadata": {
+ "id": "VYNA79KmgvbY",
+ "colab_type": "text"
+ },
+ "cell_type": "markdown",
+ "source": [
+ "Copyright 2019 The Dopamine Authors.\n",
+ "\n",
+ "Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at\n",
+ "\n",
+ "https://www.apache.org/licenses/LICENSE-2.0\n",
+ "\n",
+ "Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License."
+ ]
+ },
+ {
+ "metadata": {
+ "id": "emUEZEvldNyX",
+ "colab_type": "text"
+ },
+ "cell_type": "markdown",
+ "source": [
+ "# Dopamine: How to train an agent on Cartpole\n",
+ "\n",
+ "This colab demonstrates how to train the DQN and C51 on Cartpole, based on the default configurations provided.\n",
+ "\n",
+ "The hyperparameters chosen are by no mean optimal. The purpose of this colab is to illustrate how to train two\n",
+ "agents on a non-Atari gym environment: cartpole.\n",
+ "\n",
+ "We also include default configurations for Acrobot in our repository: https://github.com/google/dopamine\n",
+ "\n",
+ "Run all the cells below in order."
+ ]
+ },
+ {
+ "metadata": {
+ "id": "Ckq6WG-seC7F",
+ "colab_type": "code",
+ "cellView": "form",
+ "colab": {}
+ },
+ "cell_type": "code",
+ "source": [
+ "# @title Install necessary packages.\n",
+ "!pip install --upgrade --no-cache-dir dopamine-rl\n",
+ "!pip install gin-config"
+ ],
+ "execution_count": 0,
+ "outputs": []
+ },
+ {
+ "metadata": {
+ "id": "WzwZoRKxdFov",
+ "colab_type": "code",
+ "cellView": "form",
+ "colab": {}
+ },
+ "cell_type": "code",
+ "source": [
+ "# @title Necessary imports and globals.\n",
+ "\n",
+ "import numpy as np\n",
+ "import os\n",
+ "from dopamine.discrete_domains import run_experiment\n",
+ "from dopamine.colab import utils as colab_utils\n",
+ "from absl import flags\n",
+ "import gin.tf\n",
+ "\n",
+ "BASE_PATH = '/tmp/colab_dopamine_run' # @param"
+ ],
+ "execution_count": 0,
+ "outputs": []
+ },
+ {
+ "metadata": {
+ "id": "bidurBV0djGi",
+ "colab_type": "text"
+ },
+ "cell_type": "markdown",
+ "source": [
+ "## Train DQN"
+ ]
+ },
+ {
+ "metadata": {
+ "id": "PUBRSmX6dfa3",
+ "colab_type": "code",
+ "colab": {}
+ },
+ "cell_type": "code",
+ "source": [
+ "# @title Load the configuration for DQN.\n",
+ "\n",
+ "DQN_PATH = os.path.join(BASE_PATH, 'dqn')\n",
+ "# Modified from dopamine/agents/dqn/config/dqn_cartpole.gin\n",
+ "dqn_config = \"\"\"\n",
+ "# Hyperparameters for a simple DQN-style Cartpole agent. The hyperparameters\n",
+ "# chosen achieve reasonable performance.\n",
+ "import dopamine.discrete_domains.gym_lib\n",
+ "import dopamine.discrete_domains.run_experiment\n",
+ "import dopamine.agents.dqn.dqn_agent\n",
+ "import dopamine.replay_memory.circular_replay_buffer\n",
+ "import gin.tf.external_configurables\n",
+ "\n",
+ "DQNAgent.observation_shape = %gym_lib.CARTPOLE_OBSERVATION_SHAPE\n",
+ "DQNAgent.observation_dtype = %gym_lib.CARTPOLE_OBSERVATION_DTYPE\n",
+ "DQNAgent.stack_size = %gym_lib.CARTPOLE_STACK_SIZE\n",
+ "DQNAgent.network = @gym_lib.cartpole_dqn_network\n",
+ "DQNAgent.gamma = 0.99\n",
+ "DQNAgent.update_horizon = 1\n",
+ "DQNAgent.min_replay_history = 500\n",
+ "DQNAgent.update_period = 4\n",
+ "DQNAgent.target_update_period = 100\n",
+ "DQNAgent.epsilon_fn = @dqn_agent.identity_epsilon\n",
+ "DQNAgent.tf_device = '/gpu:0' # use '/cpu:*' for non-GPU version\n",
+ "DQNAgent.optimizer = @tf.train.AdamOptimizer()\n",
+ "\n",
+ "tf.train.AdamOptimizer.learning_rate = 0.001\n",
+ "tf.train.AdamOptimizer.epsilon = 0.0003125\n",
+ "\n",
+ "create_gym_environment.environment_name = 'CartPole'\n",
+ "create_gym_environment.version = 'v0'\n",
+ "create_agent.agent_name = 'dqn'\n",
+ "TrainRunner.create_environment_fn = @gym_lib.create_gym_environment\n",
+ "Runner.num_iterations = 50\n",
+ "Runner.training_steps = 1000\n",
+ "Runner.evaluation_steps = 1000\n",
+ "Runner.max_steps_per_episode = 200 # Default max episode length.\n",
+ "\n",
+ "WrappedReplayBuffer.replay_capacity = 50000\n",
+ "WrappedReplayBuffer.batch_size = 128\n",
+ "\"\"\"\n",
+ "gin.parse_config(dqn_config, skip_unknown=False)"
+ ],
+ "execution_count": 0,
+ "outputs": []
+ },
+ {
+ "metadata": {
+ "id": "WuWFGwGHfkFp",
+ "colab_type": "code",
+ "colab": {}
+ },
+ "cell_type": "code",
+ "source": [
+ "# @title Train DQN on Cartpole\n",
+ "dqn_runner = run_experiment.create_runner(DQN_PATH, schedule='continuous_train')\n",
+ "print('Will train DQN agent, please be patient, may be a while...')\n",
+ "dqn_runner.run_experiment()\n",
+ "print('Done training!')"
+ ],
+ "execution_count": 0,
+ "outputs": []
+ },
+ {
+ "metadata": {
+ "id": "aRkvG1Nr6Etc",
+ "colab_type": "text"
+ },
+ "cell_type": "markdown",
+ "source": [
+ "# Train C51"
+ ]
+ },
+ {
+ "metadata": {
+ "id": "s5o3a8HX6G2A",
+ "colab_type": "code",
+ "colab": {}
+ },
+ "cell_type": "code",
+ "source": [
+ "# @title Load the configuration for C51.\n",
+ "\n",
+ "C51_PATH = os.path.join(BASE_PATH, 'c51')\n",
+ "# Modified from dopamine/agents/rainbow/config/c51_cartpole.gin\n",
+ "c51_config = \"\"\"\n",
+ "# Hyperparameters for a simple C51-style Cartpole agent. The hyperparameters\n",
+ "# chosen achieve reasonable performance.\n",
+ "import dopamine.agents.dqn.dqn_agent\n",
+ "import dopamine.agents.rainbow.rainbow_agent\n",
+ "import dopamine.discrete_domains.gym_lib\n",
+ "import dopamine.discrete_domains.run_experiment\n",
+ "import dopamine.replay_memory.prioritized_replay_buffer\n",
+ "import gin.tf.external_configurables\n",
+ "\n",
+ "RainbowAgent.observation_shape = %gym_lib.CARTPOLE_OBSERVATION_SHAPE\n",
+ "RainbowAgent.observation_dtype = %gym_lib.CARTPOLE_OBSERVATION_DTYPE\n",
+ "RainbowAgent.stack_size = %gym_lib.CARTPOLE_STACK_SIZE\n",
+ "RainbowAgent.network = @gym_lib.cartpole_rainbow_network\n",
+ "RainbowAgent.num_atoms = 51\n",
+ "RainbowAgent.vmax = 10.\n",
+ "RainbowAgent.gamma = 0.99\n",
+ "RainbowAgent.update_horizon = 1\n",
+ "RainbowAgent.min_replay_history = 500\n",
+ "RainbowAgent.update_period = 4\n",
+ "RainbowAgent.target_update_period = 100\n",
+ "RainbowAgent.epsilon_fn = @dqn_agent.identity_epsilon\n",
+ "RainbowAgent.replay_scheme = 'uniform'\n",
+ "RainbowAgent.tf_device = '/gpu:0' # use '/cpu:*' for non-GPU version\n",
+ "RainbowAgent.optimizer = @tf.train.AdamOptimizer()\n",
+ "\n",
+ "tf.train.AdamOptimizer.learning_rate = 0.001\n",
+ "tf.train.AdamOptimizer.epsilon = 0.0003125\n",
+ "\n",
+ "create_gym_environment.environment_name = 'CartPole'\n",
+ "create_gym_environment.version = 'v0'\n",
+ "create_agent.agent_name = 'rainbow'\n",
+ "Runner.create_environment_fn = @gym_lib.create_gym_environment\n",
+ "Runner.num_iterations = 50\n",
+ "Runner.training_steps = 1000\n",
+ "Runner.evaluation_steps = 1000\n",
+ "Runner.max_steps_per_episode = 200 # Default max episode length.\n",
+ "\n",
+ "WrappedPrioritizedReplayBuffer.replay_capacity = 50000\n",
+ "WrappedPrioritizedReplayBuffer.batch_size = 128\n",
+ "\"\"\"\n",
+ "gin.parse_config(c51_config, skip_unknown=False)"
+ ],
+ "execution_count": 0,
+ "outputs": []
+ },
+ {
+ "metadata": {
+ "id": "VI_v9lm66jzq",
+ "colab_type": "code",
+ "colab": {}
+ },
+ "cell_type": "code",
+ "source": [
+ "# @title Train C51 on Cartpole\n",
+ "c51_runner = run_experiment.create_runner(C51_PATH, schedule='continuous_train')\n",
+ "print('Will train agent, please be patient, may be a while...')\n",
+ "c51_runner.run_experiment()\n",
+ "print('Done training!')"
+ ],
+ "execution_count": 0,
+ "outputs": []
+ },
+ {
+ "metadata": {
+ "id": "hqBe5Yad63FT",
+ "colab_type": "text"
+ },
+ "cell_type": "markdown",
+ "source": [
+ "# Plot the results"
+ ]
+ },
+ {
+ "metadata": {
+ "id": "IknanILXX4Zz",
+ "colab_type": "code",
+ "outputId": "e7e5b94c-2872-426b-fb69-a51365eb5fe4",
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 53
+ }
+ },
+ "cell_type": "code",
+ "source": [
+ "# @title Load the training logs.\n",
+ "data = colab_utils.read_experiment(DQN_PATH, verbose=True,\n",
+ " summary_keys=['train_episode_returns'])\n",
+ "data['agent'] = 'DQN'\n",
+ "data['run'] = 1\n",
+ "c51_data = colab_utils.read_experiment(C51_PATH, verbose=True,\n",
+ " summary_keys=['train_episode_returns'])\n",
+ "c51_data['agent'] = 'C51'\n",
+ "c51_data['run'] = 1\n",
+ "data = data.merge(c51_data, how='outer')"
+ ],
+ "execution_count": 7,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "text": [
+ "Reading statistics from: /tmp/colab_dopamine_run/dqn//logs/log_49\n",
+ "Reading statistics from: /tmp/colab_dopamine_run/c51//logs/log_49\n"
+ ],
+ "name": "stdout"
+ }
+ ]
+ },
+ {
+ "metadata": {
+ "id": "mSOVFUKN-kea",
+ "colab_type": "code",
+ "outputId": "8ec8c20a-2409-420e-a0a0-1b14907bfbed",
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 512
+ }
+ },
+ "cell_type": "code",
+ "source": [
+ "# @title Plot training results.\n",
+ "\n",
+ "import seaborn as sns\n",
+ "import matplotlib.pyplot as plt\n",
+ "\n",
+ "fig, ax = plt.subplots(figsize=(16,8))\n",
+ "sns.tsplot(data=data, time='iteration', unit='run',\n",
+ " condition='agent', value='train_episode_returns', ax=ax)\n",
+ "plt.title('Cartpole')\n",
+ "plt.show()"
+ ],
+ "execution_count": 8,
+ "outputs": [
+ {
+ "output_type": "display_data",
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7EAAAHvCAYAAACCOFj3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3WdgVHXWBvDnzkwmk94bCSE9IY0E\ngjRRqr0gLKIIrmWXteuu7qrrq7Kvu/Z1fRV114aKBRUFUVFQF5EupJFCGkmA9Ex6Mslkyn0/pCDS\nUmbm3pk8vy+7ZiaTE7gkc+45/3MEURRFEBEREREREdkBhdQBEBEREREREQ0Vk1giIiIiIiKyG0xi\niYiIiIiIyG4wiSUiIiIiIiK7wSSWiIiIiIiI7AaTWCIiIiIiIrIbTGKJiIgsSBRFrF27FldccQUu\nvvhiLFiwAKtXr0ZHR8ewXqe8vBwHDhwYcRxVVVVITEwc8ecTERHJFZNYIiIiC3r++eexZcsWvPXW\nW9i6dSs2b94Mg8GAP/zhDxjOavbvv/9+VEksERGRo1JJHQAREZGjaG1txbp167Bx40YEBQUBAFxd\nXfHYY49h9+7d6OnpwcMPP4zDhw/DYDDg4osvxoMPPggAWLlyJSZPnoxt27bh8ssvx9q1a+Hk5IT2\n9nbExcXhm2++gbe3N7Kzs6HRaLBmzRpERESgtbUVjz/+OIqKiqBUKrFo0SKsWrXqpLhEUcQrr7yC\nL7/8Er29vZg/fz4efvhhKJVKm/8ZERERjRYrsURERBaSm5uL4OBgREdHn/RxZ2dnzJs3Dx999BG6\nurrw7bffYuPGjfj8889x8ODBwefl5+fj66+/xl133YWFCxfixhtvxEMPPQQA2LNnD2644QZ8//33\nmD9/Pp577jkAwAsvvAAvLy9s3boVH374IT766KOTXhMAvvjiC3z77bfYsGEDvvvuOxw/fhwfffSR\nlf80iIiIrINJLBERkYW0trbCz8/vjI/fcsstePXVVyEIAry8vBAbG4uqqqrBxy+88EIoFKf/1Rwd\nHY20tDQAwMUXX4zs7GwAwI4dO7B8+XIAgLe3NxYuXIjdu3ef9Lnbt2/HkiVL4OHhAZVKhaVLl2Lb\ntm2j+l6JiIikwnZiIiIiC/Hx8UF9ff0ZH6+srMTTTz+N8vJyKBQK1NXVYfHixYOPe3l5nfFzf/mY\np6cn2tvbAQDNzc3w9PQ86bGGhoaTPrejowNvvfUWPv74YwCAyWSCr6/v8L45IiIimWASS0REZCFp\naWloampCQUEBkpKSBj9uMBiwZs0a5OXlISkpCa+88gqUSiWuu+66Ib92a2vr4P9va2sbTGr9/f3R\n2tqKcePGDT7P39//pM8NDAzEvHnzsGLFitF8e0RERLLAdmIiIiIL8fT0xO9+9zs8+OCDOHr0KACg\nu7sbjz32GAoLC9HU1ISJEydCqVRi9+7dOHr0KHQ63WlfS6VSnbSWp6KiAoWFhQCArVu3YsqUKQCA\nOXPmDFZYm5ub8d1332HOnDknvdb8+fPxxRdfoLu7GwCwfv16bNy40aLfOxERka2wEktERGRBd999\nN7y8vHD77bfDZDJBoVBg/vz5WL16NbZv346nnnoKr776KubPn4+77roLL730EiZOnHjK68ydOxcP\nPPAAqqurMWfOHKSnp+Odd97BwYMH4erqitdeew0AcN9992H16tW45JJLoFAosGrVKqSmpp501nbB\nggUoLS3FNddcAwAIDw/HP/7xD9v8gRAREVmYIA5naR0RERHZ3Oeff47NmzfjnXfekToUIiIiybGd\nmIiIiIiIiOwGk1giIiIiIiKyG2wnJiIiIiIiIrvBSiwRERERERHZDSaxREREREREZDfscsWO0WhC\nS8vp9+oROQofH1de5+TweJ3TWMDrnMYCXudkaQEBHmd8zC4rsSqVUuoQiKyO1zmNBbzOaSzgdU5j\nAa9zsiW7TGKJiIiIiIhobGISS0RERERERHaDSSwRERERERHZDSaxREREREREZDeYxBIREREREZHd\nYBJLREREREREdoNJLBEREREREdkNJrFEREREREQ0bL29vfjmm69s/nWZxBIREREREdGwlZQU49tv\nt9j866ps/hWJiIiIiIjIqr78chPWr38fJpMJfn7+ePTR/4WPjy/+/vfHkZeXi8jIKMTFJaC5uQmP\nPLIaDQ31eP75p3Hs2FEAwL333o8ZM2ahtrYGt912M1asuBlffrkR7e3tuPvuPyI9fQoeeeQBdHV1\n4Y47fodXX33TZt8bK7FEREREREQOpKWlGf/617P4179ewfr1GxEaGoZ33nkTX321CVptIzZs+BIP\nPvg/2LLly8HP+cc/ViM2Ng7r13+O55//PzzxxGNoa2sFALS2tkKhEPDeex/jnnvuxxtvvAZfXz/8\n4Q93ISkp1aYJLMAkloiIiIiIyKH4+Phi69YdCAwMAgBMmpSOmppq5ObmYO7c+VCpVAgODsGMGbMA\nAN3d3cjKOohly5YDAMLCxmPSpDTs2bMLAGAymXDZZVcBAOLjE1BfXyfBd3UC24mJiIiIiIgciMlk\nwptv/hu7d/8Ek8kEnU6H8ePD0dHRDg8Pz8HnBQQEoqGhHl1dnRBFEbfddsvgY93d3Zg8eSoAQKlU\nwsXFBQCgUChgNptt+w39CpNYIiIiIiIiB/LDD99h9+6fsGbNG/D29sbmzRuxbds3cHNzQ3d39+Dz\nmpq0AABvbx8olUq8+eY6uLq6nvRatbU1No19KKzaTvzss89i2bJlWLJkCbZt24ba2lqsXLkSy5cv\nx7333ove3l4AwObNm7FkyRIsXboUn376qTVDIiIiIiIicmitrc0IDg6Bt7c32tpa8d//fofu7m5M\nnJiEHTv+C7PZjPr6OuzbtwcAoFKpMGPGLGza9BkAoKenB08++bdztg2rVCrodH1VXFuyWhK7b98+\nlJaW4uOPP8abb76JJ598Ei+99BKWL1+ODz/8EBMmTMCGDRug0+nwyiuv4J133sG6devw7rvvorW1\n1VphERERERERObQFCy5GW1sbli1bhNWrH8Hvf38HGhrq0dSkhVqtxrJli/DCC89g/vyLIAgCAOCB\nBx5GTk4Wli9fgltuuQHjxoUiKCj4rF8nNTUNWq0WixZdApPJZItvDQAgiFZKm00mE/R6PVxdXWEy\nmTBz5ky4ubnh22+/hVqtRnZ2Nt5++20sX74cn332GZ5//nkAwGOPPYY5c+Zg3rx5Z339xsYOa4Tt\nMERRRGtnL3w8nKUOhUYoIMCD1zk5PEe+zjt0vWjp0FvktZydlAjydT33E22otVMPTzc1FP1vfhxN\nQ4sOPb2WeUPm4+OGlpYui7yWHAX5usLZSSl1GLJmMpvR2W2El5ta6lAGiaKI+pZu9Bp4nQ/FOH83\nqJSOMRNXFMXBxPWVV/4PJpMR99xzv8RRnSogwOOMj1ntTKxSqRzsp96wYQMuuOAC7Nq1C2p13z9e\nPz8/NDY2QqvVwtfXd/DzfH190djYaK2wxox9BfV446tC3LAwDvOnhEkdDhHRmGIwmvDIG/vR2W2w\n2GvetTgFk+MCLPZ6o1HV0InVaw8gLdYfty9KglLhGG/sBny5pxIbfyqXOgy74eWmxnXzY3HexMDB\nN8Z0Qm1TF/7zRQGqGruw8uI4XJgWKnVIMJrMeG9rMXYdqpU6FLsxzt8Nf74+XVY3IkZi164dWLv2\nTbz22lswGo3Yu3cXbr55ldRhDZvVBzt9//332LBhA95++21cdNFFgx8/UwF4qIXhs2XmBGzPzQIA\nfPzfMkxNDkF0mLfEEdFI8DqnscARr/ODh+vR2W3AxAhfxI4f3c9fg9GMb/ZWYt/hBlw8K8oyAY7S\npj2VMIsiskoasX77EdxzbToUCsdIXjKL6rFpZzn8vV0wMyVE6nBkr6fXhB8zj+M/mwuw/3ADbluS\nitAAd6nDkgVRFPHdz8fw+qY86HtNcFYr8e63xeg1A9dfFC9Zwt+tN+KZ9w4gs6gBUeO8kBztJ0kc\n9qShRYd9+XV44ZMc/OP2WfDx0Egd0ohdddWlyM7+GTfeeC0UCgXmzJmDa69dBIWd3Yy0ahK7c+dO\n/Pvf/8abb74JDw8PuLq6oqenBxqNBvX19QgMDERgYCC0Wu3g5zQ0NCAtLe2cr+2o7WeWcLSuA2XH\nWxHk64r6Zh2eeudnPHbTVLg4cxi1PXHkNkuiAY56ne/IPA4AuHpWBOJGmcQCwOGKJmQVNaCsQgsv\nd2mPiRhNZmw/eBzuLk4I8NbghwPHoQSwbF6M3VfhGlu78dy6g1AqFLj96iREhnie+5OGwFGv8wHz\n0sfhg20lyCltxF3P/ReXTpuAy2dMgHoMtxjregx499tiHChqgKuzCncsSsb4QHe88EkOPtpWjKq6\ndtx4SbzNuxjau3rx4qe5qKzrQEqUH25flASN2jLvDx35OhdFER4aJ3x38DgeWrPL7iuyd9/9Z9x9\n958H/7upSZ5t4Ge7yW21fzkdHR149tln8Z///Afe3n2/wGfOnImtW7cCALZt24bZs2dj0qRJyMvL\nQ3t7O7q6upCVlYWMjAxrhTUm7MjtG4N93bwYXHJeOOpbuvH+tmKbTw0jIhqLRFFETmkj3F2cEB1q\nmSRoZnIwzKKIfYX1Fnm90civaEaHzoBpiUG4b+kkhPi5YtuB4/h671GpQxuVXoMJr27MR1ePESsu\nirNYAjsWBHq74L6lqbhjUTI8XNX4ck8lHnvrZ+SVN0kdmiTKqtrw+NsHcKCoAbFhXvjbLechIyEQ\nQb6u+OvKDEwI9sDOQ7V4+bO+Cq2t1Lfo8OS6TFTWdeD8lBDcvSTFYgmsoxMEAdfNj8HCjPGo0Xbh\nuY+y0d7VK3VYY5rVktgtW7agpaUF9913H1auXImVK1fitttuw6ZNm7B8+XK0trZi0aJF0Gg0uP/+\n+3Hrrbfi5ptvxp133gkPD8drLbOVnl4j9hXUwdfTGSlRflh8YRQiQzyxt6Aeu/POPiKbiIhG72h9\nB1o7e5Ea7WexKsu0xCAoFQL25Ev/c3wghpnJwfBwVeP+ZWnw83TG5z+VY3tWlcTRjYwoinh/WwmO\n1ndgdmoILpg0TuqQ7I4gCMhICMTffzcNF00dD21bD/71SS5e3ZhnsQFncmc2i9i8uwJPf5CF5o4e\nXDUrAn9Zng4/rxOtp15uajy4PB3Jkb44dKQJz36UjXad9ZOhitp2PLkuEw2t3bhyZgRuvizBYYYU\n2cpAIrsgI4yJrAxYbTqxtTlqu8Jo/ZRbg3e+KcKi8yNx1fmRAPrao1avPQCT2YzHb5qKED83iaOk\noXDkthyiAY54nW/aWY7Nuytxx6JkZCQEWux1X/7sELJLtVh981SEB0lzs7erx4A/vrwLgT6ueOLW\n8wbbh+uadXj6/Ux06AxYdVUSpiUGSRLfSO3Iqca73xZjQrAH/rpiMpxUlm2DdcTr/FyON3Tiva1F\nOFLdDme1EtecH4n5GWEONwRsQHN7D17/shAlx1vh6+mMVVcmnfUogdFkxrvfFGF3fh0CfVzwp2sn\nIdDHOhPIDx3R4tVN+TAYzVh5UTzmpFtnsNRYuc5FUcRHP5Ti+4NVCO0f9uRpx63FciZJOzFJY0dO\nDQQBOD/1xDCKAG8X3HRpAnoNZry2qcBio9SJiOhUOWVaqJQCkiJ9z/3kYZiZ3LerT8pq7IHDDTCa\nRMxMDj7p/Guwryv+tCwNGmcl3vyqEIeOaM/yKvJSUduOD74rgZtGhTsXJVs8gR2rxge64+EVU3DT\npQlQKQSs/28Z/vedgyirbpM6NIvLLG7E42//jJLjrZgSH4C/3XLeOc/Cq5QK3HL5RFw+YwIaWrrx\n5LpMVNS2Wzy2nbk1eGlDHkQRuOuaFKslsGOJIAi4fn4sFkwJQ7W2C8+tt001nU6mXL169WqpgxgJ\nHS+WUxyr78CmnRVIi/E/5YdUqL8b2jr1OFTehK4eIybF+EsUJQ2Vm5szr3NyeI52nTe39+DTH48g\nMcLX4i2pAd4u2J5VhZomHRZODZNkP+sH35egtUOPWy9PPGVYoJe7M2LDvLGvsB4HixoQN977pDZK\nOerQ9eL59dnQ9Rhx1+IURFjpHKyjXedDJQgCJgR7YHZqCDq7Dcgvb8bOQ7Vo6ehBTJi33Q9+0htM\n+PC7Enz64xEIAG64KA5L50QP+fsSBAGJEb7wcHXCwaJG7Cusx4RgDwRZoCIriiK+3FOJ9T+UwVWj\nwp+WpVn8xtqvjaXrXBAEJEf5oqvHiNyyJuSVNyEjIdBh9yXX1tZg8eLL8fPPe7Fly5fYunULAgIC\nMW5cKPT6Hrz44nN4443X8PXXX2Dv3t2YNCkNbm5uqK2tweWXz8fs2XPg69s3BXvLli9RWlqC2Nj4\nc35dN7czDzJkJdaBDAx0uiDt9G+crpsfi7AAN2zPrsbBogZbhkZENCbklvVVIK1xo9BJpcB5iUFo\n7+pFQUWzxV//XOqbdThS3Y7ECB/4eJz+jUXceG/csSgZJrOI/9twCMfq5dtaaDaLeH1zAZra9bh6\ndiSSo7hmxFo8XNW45bKJeOiGyQgNcMNPubX46+v7sPNQDcz2eaoNVQ2deOLdg/gxpwZhAW549Kap\nmJMWOqIJ3fMmh+GOa1JgFkW8tOEQdueNbneryWzGuq3F2LSzAn6eGvx15RTEhHqN6jXpVIIgYPmC\nWMyfEobqxi48b6PzzVIJD5+ANWtex5o1r+Mvf3kEL774HMrKSvHyyy/Cz88fa9d+iDfeeA/XX78C\n999/N4xGIwAgIiIS//73yxaPh0msg9D3mrCvoA4+Hs5IiTr9nTa1kxK3XZ0MtZMCa78pQmNrt42j\ntByjyezQPyiIyD5l9yexaVbqdpmV3HdURIqW4hMDnc6+O3VSjD9uvWIievRGvPBJLuqbdbYIb9g2\n7SpHQWULUqP9cMXMCKnDGRPixnvj8Zum4tq5MTAYzVi7pQjPfJCFqsZOqUMbMlEU8UNmFf733YOo\n0XZh/pQwPPrbDIT6j27eyJT4ADxwXRo0aiXe+vowvtpTOaKtEnqDCa98no8fc2oQHuiOR26cwlko\nVjSYyE4OQ1V/ItsxBt6fhoaG4cYbb8Gnn36Efft2Y+XKmwcfS01NQ0JCInbt2gEAiI+fCBcXV2Rm\nHrBoDJyr7SB+PlyPbr0JCzPGn3Vowjh/N9ywMA5rtxThP5sL8NANk+1qOl1tUxd25tZid34tdD1G\n/PHaSUiMsG57DBHRUHTrjSg62oLwQHertdFGhngg2NcVWSVa6HoMcNU4WeXr/JpZFLEnvw7OaiUm\nxwWc8/nTE4Oh6zHi/W0leH59Dv66csoZq7dSyCnV4qs9RxHgrcHvr0yUpDV7rFIpFbhkWjjOmxiI\nD78vRVZJI/629gAWTh2Pq2ZFyHrlS4euF2u3FCGnTAt3Fyfcclky0mItd8MqNswbD6+Ygn99koPP\nfypHS4ceNyyMg0IxtOuzQ9eLlzYcwpGavo6JO69JOaXtnyxPEAQsXxgLESL+m1WN5z7Kxp+vT4eH\nq+WHPX3y3zIcsHA35dSEQFw7L2bYn5eQMBEvvPAsEhOToFKdfJ3Fxsbj6NFKxMdPBACsWnUH/v73\nx/Hvf79tkZgBVmIdxo7cvoFOs1PPfQbr/JQQTE8KQnlNOzb+VG6D6EZH32vCrkO1eOr9TDzyxn58\n+/MxDNycfG1TPupb5HmXn4jGlsLKZhhNolVnDgiCgFkpwTCazBZ/I3M2pcdb0dTeg4z4ADirh3bm\na97kMFwzOxJN7T3458c56Ow2WDnKoalv0eGNrwrhpFLgzmtS4GajGwF0Ml9PDe5anIJ7fpMKHw9n\nfLv/GP7nzf3YnVcLs1l+LcaHK5vx+Ns/I6dMi4kTfPC3W86zaAI7YJy/G/66MgPjA92xPbsar2zM\nG9JAzsbWbjz5fhaO1LRjRlLfDmcmsLYjCAJuWBiHuZNDUdXYhec+ynH4iqxOp4MommEymU95TBTF\nkzoJxo8PR1xcAn74YZvFvj6vbgdwrL4D5TXtSI32G9Ldf0EQsPKieJTXtOOb/ceQMMEHKTI7CySK\nIirrOrAztwb7CuvR02uCACApsm9YSlqMP/YX1uPtLYfx0oZDeGRlBlw1vJyJSDo5pf2txFZ4Y/tL\n0xOD8fmOcuzJr8OFabaZNDqwZ3zWOVqJf+2KmRHo7Dbiu4PH8a9PcvHAdWmSvrHua7XMQ7feiFsv\nnyjZqiI6IS3GHxMn+OCrPZXY+vMxvPX1YWzZdxTXzI7C5PgAyavkRpMZX+yqwJa9RyEIApZcGIVL\np00YcnV0JHw8nPHg8sl4ZWMesku1eH59Du75TSrcXU5/w+VoXQde/DQXbV29uHR6OJZcGC35n9tY\nJAgCViyMA0Rge3Y1nl+fgz9fn37Gv7eRuHZezIiqptZQVFSIyy+/Ctu3/wCDwQAnpxPfZ1lZCWbN\nuuCk59988+/wpz/djcWLl55SuR0JVmIdwE/9A50uPMNAp9NxcVbh9quToVIKePOrQtksIu/qMeCH\nzCqsXntgcGCCi7MKV82KwDO3zcD9y9IwNSEQTioFzk8NwcXnjUdtkw7/3pwvyzu3RDQ2mM0ico80\nwdtdjQnB1k2M/Lw0SJjgg9KqNjTYoBNFbzDhQHED/Dw1iAs/+9qQXxMEAcvmx2BWcjAqatux5vM8\nGIyn3rW3BVEU8d63Rahq7MLc9FDMShleQk7W4+ykxJILo/HUqhmYnRqC+uZuvLopH//7zgEcOqId\n0dnQ0TKZzcguacRT72fi671H4eelwcMrJ+PyGRFWTWAHuGpU+OO1kzAtMQhl1W146v1MaNtOnWVS\nUNGMpz/MQntXL5YviMXSOTFMYCUkCAJWXBSHuemhON7Qiec+ypZNF4olVVdXYf36D3HttTdg6tRp\nePvt1wcfy8vLRVFRIWbNmn3S5/j6+mH27AvxxRefWyQGlq7snN5gwt6COni7q5EaPbxq6oRgDyyd\nG4OPvi/Fm18V4v5laTb5wfxrZlFE8bFW7DxUg4NFjTCazFAqBEyJC8DsSeOQHOl7xriWzolBjVaH\nvPImfLK9DNfNj7Vx9EREwJGaNnR2G3Bh2jibvIGcmRyMw0dbsCe/DotmR1n1a2WVNELf2zdzYSTf\nm0IQcNNlCdDpjcgu1eL1zQW4bVHSWec3WMP27GrsLahH1DhP/q6QKT8vDW6+bCIunT4BX+yqwP7C\nerz46SHEhHph8QVRSJjgY/UYWjr02Jlbgx25NYM3+KcnBWHlRfE27yJQKRX4/ZWJ8HF3xrc/H8M/\n1mXij0snDXYQ7M2vw9tbDkMQBNy+KBkZCYE2jY9OTxAE3HBRHEQAP2ZX4/mPsvGAhSuyUjh27Cju\numsVDAYDzGYT7r//LwgODsb99z+IZ599EtdfvxjOzhoEBgbhmWf+ddpq6/XXr8SmTZ9ZJB5BlOL2\nlgU0Nsp3bL8t7TpUi7e3HMaVMyNwzQXDfyMjiiLWfN7XrnLN7EhcOSvSClGeXmunHrvzarEztxYN\n/ZOSg31dccGkcZiRHAwvt6EdiNf1GPGPdQdR26TDzZcmYLaFdzNKJSDAg9c5OTxHuc4/3V6Gb/Yf\nw72/SbXJHu6eXiP++PJueLg64enbZlg1cf7nxzkoqGjGk6umI9h35PsrDUYT/vVJLoqOteL8lBDc\nfFnCiNaRjERZdRue+SALLs4qrL55Knw9bbu/1lGuc1s73tCJjT+VI6d/6ndihA8WXxCNqHGW3edr\nFkUcPtqCH7OqkV2qhVkU4axWYmZSMOakh2J8oLtFv95IfHfgONb/UApntRJ3LU7B0boOfPrjEbg6\nq3D3khTEh1s/wT8XXucnM4si3t9a3DcpOsgdD1xn/4ns2ej1eixbtghr134AHx/LDF0NCDhzZ5Ny\n9erVqy3yVWxsrCxTPpd124rR2qHH765IHNGZUEEQkBTpi/2H6weHFVhzOb3JbMahsr6q6bqtJSis\nbIHBaMb0xCDcsDAOS+dGIzbMG5ohDg4B+nYnJkf5Yl9BHTJLGpEQbt3vwVbG0tJwGrsc5Tr/8PsS\n9BpNuPHieChtMPFdpVSgrlmH4uOtSIzwtdrPvJYOPT74rgTR4zxx2fQJo3otpUKByXEBKKxsxqHy\nJugNJiRF+Fo9kW3v6sXz63PQ3WvEPUtSJTkH6yjXua15uakxLTEIyVG+aGrrQWFlC37KrcGx+g6E\n+rvBc4g3u8+ks9uA/2ZV4+2vD+O7g1WobdIhLNAdi86PxK2XT8SU+MAh31C3tuhQL4T4ueJgcQN2\n59WhoLIFPh7O+Mv16YgcJ48dsLzOTyYIAlKi/dDe1YvcI00oqGxGRkIg1E5Df49rT1QqFXx9/fD0\n009Aq9ViypSpo35NN7czT7VnEmvHqho68flP5UiN9sO8yWEjfh21kxKRIZ7YnVeH/IpmzEoJsfg/\nsLauXmz7+Rje+LIQO3JrUNesQ3iwB66aFYlbL0/EtMQg+HlpRvxmxt3FCZEhnthbUIecMi2mxgfa\nbPWEtfCXAY0FjnCd1zfrsGlXBVKj/c65Q9WSNGol9uTXQaEA0mLPvfZmJH7MrkZBZQuumBmByJDR\nV7+cVH2JbG6ZFjllTXBSKRA3fnjnbIfDZDbj5c8OoaqxC7+ZE23Tv59fcoTrXEq+HhrMTA5B/Hhv\n1Ld0o7CyBT9mV6OuWYfxAe7Dqm6JoogjNe34bEc51m4pQl55E/QGc1/L8MXxWHxBFCJDPOGkkt/Y\nmNAAd8SN90ZWiRZBvi74y/XpCBpFd4Sl8To/1UAi29bVi0NjIJGNjo7F4sXXWiSBBZjEOqwv91Si\norYd186NGfUiaz9PDQSFgOxSLWq0XZiWGGSRu+PlNe3Y8GMZ3vmmCIVHWyAIwAWTxuG3lyTg6vMj\nLfqLIsDbBR6uahwoasDhoy2YkRxsVztwf42/DGgscITrfHdeLQoqmnHptHCrD3X6JT9PDXYeqsWx\n+o6+HeEW/nkniiLe3VqMbr0Rt1w20WJvupydlEiP9UdmcQMyS7TwdFNbJEE+nc9+PIJ9hfWYHBeA\n5Qtibda+/GuOcJ3Lgb+3C85PDUHUOE/UaLtQWNmC7VnVaGrvQXigx1k70rr1RuzKq8U73xTh671H\nUdXYiQBvDa6YGYHfX5mI6UkJTZQVAAAgAElEQVTB8PUc+c10W/H3csH8KWGYPyVMdjfreZ2fniAI\nSI32Q2tnXyJbWNmC6UlBdv0e1VbOlsRysJOd0htM2JtfBy93NVJjLLMe5/LpE1B0tAW5R5rw/cEq\nLJw6fkSvM7C/8IfMKpTXtAMAQvxcsWBKGGYkB1t1kfnc9FBUN3biv1nVeOPLQty5OIVT+ojIqnJK\ntRAApNrgLOwvKRQCZiYH4+u9R5FdqsW0xCCLvv7R+g7UaLuQER9g8XNcvp4a3H9dOp56PxPvby2G\ni1ppsZunAw4WNeCb/ccQ5OOCWy6bKPvkhIamLyHwR3KUHzKLG7FpZzl2HqrF3oK+lVNXzJgAL/cT\nb3yPN3T2D/Wqg77XBIUgYEp8AOamhyJhgo9dvkcY6q5mkg+FIODGS+LRazRhX0E9csuaLP4ze6xh\nEmunDhY1QKc34oopERab8KhQCFh1ZSIef/tnfLK9DLHjvRARPPS7462devyYXY0fc2rQ3tULAX37\n3+ZnhCFxgo/N3kBcNz8WtU06ZJdqsfGnciy5MNomX5eIxp7ObgNKq9oQNc5TkrNzA0ns7vxai78h\n2tO/G9ZaLbjBvq7407VpePajLLz+ZSE+/L4UESEeiAj2RGT///p4nPku/NnUNnXh7S2HoXZS4M7F\nKdwj7oAUgoCpCYGYEheAvQV1+GJXBX7IrMLOQzVYMGU8QvxcsSOnBmXVbQD6dq9eOi0cs1PHjfi6\nIhoNRX9Fdl9BPXQ9jrd2x9b4U91O7cipgQDgglTLvrnwcnfG765MxAsf5+Lfmwrw+M1TzzpSXhRF\nlNe044fMKhwoaoDJLMLFWYWLzxuPuZPDEOjtYtH4hkKlVOD2Rcn4+3sH8fXeoxjn74YZScE2j4OI\nHF9eeRPMooi0WNtWYQeE+LkhMsQTBRXNaO3Uw9vdMm/OjSYz9hXWw8PVCclRlpkyeToTgj3wwHXp\n+Gb/MVTWtiO/vBn55c2Dj3u5qxEZ7DmY3EaEeMDT9ew3C3p6jXhlYz56ek1YdVUiwgKknyxL1qNQ\nCJiVEoJpiUHYmVuDzXsqsWXfUQCAACA5yhdz00ORGu1n87VORL820I3Y02uSOBL7xyTWDlU1dqKs\nug3JUb7wt0KSmBzph8umT8CWfUfx3tZirLoy8ZQqqsFoxoGienx/sAqVdX3j1EP93TB/ShhmJAVL\n3uri7uKEe3+Tir+/l4m1W4oQ6OOCaJlM7yMix5FT2rf6I83GrcS/NDM5GBW17dhXUI9LpoVb5DXz\nypvQ2W3Agowwq5/bigzxxB2LkgH0VbYra9tRUdeBytp2VNZ1IKdMO7hiBeg7C9yX1HogIsQTEcEe\ncOs/GyiKIt75pgg12i4syAjD9ETewBwrVEoF5k4Ow6yUEPyUW4OuHiNmJAdLcjOd6Exc+t8fd9tZ\nEnv8+DG89NI/0draApPJjJSUVNx5531YuHA2UlImDT7v//7vNSiVSmRnZ+LRRx/Cww8/hlmzZlsl\nJiaxduinnBoAwIWTQq32NRbNjkTx8RbsL6xH4gSfwd2rLR16bM+uxk851WjXGSAIQHqsPxZMCUOC\nDVuGhyLEzw23XZ2EFz/NxZrP8vDobzNsvhuQiByX0WRGfkUT/L00GOc/uuF6ozEtMQjrfyjFnvxa\nXHzeeIv8HN6T39dKPMvG03zdXZyQHOWH5KgTsx5aO/Wo/EVSW1HbjsziRmQWNw4+J9DHBRHBHlCr\nlPj5cANiwrxw7dwYm8ZO8qB2UmJBxshmehBZ24lKrFHiSIbOZDLhf/7nL7jvvj8jPX0KRFHEiy8+\nh7Vr34C7uzvWrHn9pOdXV1fh448/OCm5tQYmsXam12DCnvw6eLmpMclCA51OR6VU4A9XJWH12wfw\nwXclcHJSIKdUi8ziRpjMItw0KlwyLRzz0kOtUg22lJQoPyybF4v1P5Ti5c/y8NCKyXB20LHmRGRb\nxcdb0a03YVZKiKQ38NxdnDApxh9ZJY043tA56j2ond0G5JZpERrghvAg6Vtxvd2dkRbjPFjtFkUR\nLR16VPQntQPJ7c+HGwAAnm5q3H51Mid/EpHsaPorsfbUTnzgwH6Eh0cgPX0KgL7hanfccQ8EQYHN\nmz8/5fl+fv74xz+ew9NPP2HVuJjE2pmDxX0DnS6fPMHqv6D9vVxw82UJeGVjPl7fXAgACAvoaxme\nnhRsN8ngwowwVDd2YuehWrz19WHcfnWSrCrGRGSfBlqJ0yVsJR4wMzkYWSWN2J1XN+ok9sDhehhN\nImYmB8vyZ6UgCPD11MDXU4Mp8YEA+hLbxtZuHK3vRHigOwf3EJEsaZxHdyb287KvkN2QZ8mQkB6Y\ngsUxV5zx8WPHKhEbG3fSx5yd+zobe3t7sXr1I6ivr8WFF87DddetgEZjm65HJrF2ZnCgU397r7VN\niQ/Eb+ZEo6qhExemjUPceG9Zvqk5G0EQsPLieNQ363CwqAFf+rvhqvMjpQ6LiOyYKIrIKdXCxVmF\n2PHeUoeD1Gg/uLs4YX9hHZbOjR7VTc49+XUQBNjVeVJBEBDo44pAH1epQyEiOqMTlVj7aScGBJjN\n5tM+cued9+Kiiy6DIAi4887fIy1tMhISEm0SFZNYO1Kt7UJpVRuSI30RYMMW3sumT7DZ17IWlVKB\nOxan4O/vHsSmXRUY5++GjIRAqcMiIjtV3diFpvYenDcxUBZtqyqlAtMmBuGHrCoUVDRj0girw7VN\nXThS047kSF9WM4mILEytUkAQRl6JXRxzxVmrptYwYUIEPvvsk5M+1tvbi6qqY1i06DeDH8vImIoj\nR8pslsRK/5uXhmxwoFOabaqwjsbTVY17lqTCWa3Em18V4mj/VGUiouHK7p+WK9VqndOZmdJXOd3d\nP5RpJPYWDOyGtZ8qLBGRvRAEARq1Cj16+zkTO3XqNNTX12LXrp8AAGazGa+99jLeffdtrF79CERR\nhNFoRF5eLiIjo2wWF5NYO2EwmrAnvxaebuoR32EnICzQHauuTITBaMZLnx1CW6de6pCIyA7llGqh\nVAhIibLegL3higj2QIifK3JKtejqMQz7882iiL35ddColUiPC7BChEREpFEr7aqdWKFQ4J//XIPN\nmzfi1ltX4o47fgd3d3c8/vjfERgYhN///re4/fZbMX36LCQmJmPPnl24665V2L9/L/7znzX44x/v\ntEpcbCe2EweLGtHVY8TlM6w/0MnRpccGYMmcaGz48QjWfJ6HvyxPh5PKPoZUEZH02jr7JuMmhHsP\n7ieVA0EQMDM5GJ/tKMeBogbMSRveGrbiY61oatfj/NQQuxncR0RkbzRqJTp0w7/RKCV/f388++y/\nTvn4HXfcc8rHZs48HzNnnm/1mJgN2YkdOdUAMLivlUbn0mnhmJEUhCM17Xjnm2KIoih1SERkJ3KP\nNAEA0mLlV62ckRQMAcCevOG3FO/JqwUAzGIrMRGR1WjUKrtasSNXTGLtQI22CyVVbUiK8EGgjHey\n2hNBEHDTpQmIGueJvQV1+O5gldQhEZGdGFitk2bFXd0j5eupwcQIH5RVt6G+RTfkz9P3mnCwuBH+\nXhpZTFsmInJUGrUSRpMZRtPpJ/7S0DCJtQM/5Q4MdBpeaxidnZNKibsXp0CjVmJ7drXU4RCRHdAb\nTCisbMY4fzfZrnOZlRwCYHjV2KySRugNJsxICobCztaoERHZkxNrdliNHQ0msTJnMJqwO68Wnq5O\nspqC6Si83J0RE+qF+mYd2nW9UodDRDJ3uLIFvUYz0mQ8YG9yXACcnZTYW1AH8xCPSuzO72slHphw\nTERE1qFR940k6tHbz3AnOWISK3OZxX0DnWalhnCgk5XEhHkBAI5Ut0kcCRHJXU5ZIwB5rdb5NWe1\nEhnxAdC29aD0eOs5n9/c3oPDlS2ICfVCkEyry0REjkLjzEqsJTArkrkd/bthL+BAJ6uJDe1LYsuq\nmMQS0ZmZRRG5ZU3wcHVCVIin1OGc1cyUvpbioeyM3VtQBxGswhIR2QLbiS2DSayM1TZ1ofh4KxIj\nfHh33Ioix3lCIQgoYyWWiM6isrYDbV29mBTtD4VC3udG48O94efpjINFDdAbzvxGSRRF7Mmvg0qp\nwHkJgTaMkIhobBpsJ7ajXbFyxCRWxjjQyTY0ahXGB7qjorYDBiMnxRHR6dlDK/EAhSBgRnIwenpN\nyC5pPOPzKus6UNukQ3qsP1xltPOWiMhRsRJrGUxiZcpgNGN3Xh08XJ2QbgdvmOxdTKgXjCYzjtZ3\nSB0KEclUTmkTVEoFkiJ8pQ5lSGYk9bUH7zlLS/HABOOZ3A1LRGQTLv2V2G5WYkeFSaxMZZY0oLPb\ngPNTONDJFgaGO/FcLBGdjra1G1WNnUiM8IFz/110uQvxc0PUOE8UVDajpUN/yuNGkxn7D9fD09UJ\nSZH2kZgTEdk7VmItg9mRTP3EgU42FTuQxPJcLBGdRk6ZFgBkvVrndGYlB0MUgX2Fp1ZjDx1pQme3\nAdOTgnmzlIjIRjid2DL4W0uG6pp1KDrWiokTfBDky4FOtuDrqYGPhzPKqlohDnGvIhGNHbn9Sewk\nO0tip04MgkopYE9e3Sk/2wbajNlKTERkOxzsZBlMYmVooAp7YRqrsLYUG+aFdp0Bja3dUodCRDKi\n6zGi6FgrJgR7wMfDWepwhsXdxQmTYvxRre3CsfrOwY93dhuQW6ZFWIA7woM8JIyQiGhsYTuxZTCJ\nlRmD0YxdebVwd3FCemyA1OGMKdH9+2JLeS6WiH4hv6IJJrOIdDurwg4YqLTuzq8d/Nj+wnqYzCKr\nsERENjaYxOqZxI4Gk1iZyS5tHBzo5KTiX48tDZyLPcJzsUT0C/baSjwgJcoP7i5O2F9YD6Opb43Y\nnvw6CAIwPSlI4uiIiMYWthNbBrMkmdkxMNCJrcQ2Nz7QHWonBUqZxBJRP5PZjENHmuDj4YzwIHep\nwxkRlVKB6YlB6NAZkF/ejNqmLlTUtiM50g/e7vbVHk1EZO/YTmwZTGJlxGgyo+hoCyJDPBHMgU42\np1QoEBXiiZrGLuh6DFKHQ0QyUFbVhq4eI9Ji/SEIgtThjNjMlIGdsbUc6EREJCGVUgGVUsFK7Cgx\niZWRbr0RImB3g0McSUyYN0QAR2rapQ6FiGQgu7Svldhez8MOmBDkgVB/N+SUabErrxYuzkqkx9r3\n90REZK80aiUrsaOksuaLl5SU4I477sBNN92EFStW4J577kFLSwsAoLW1FWlpafjDH/6AK6+8EsnJ\nyQAAHx8fvPTSS9YMS7a69X13ZFydrfrXQmcR84vhTilRfhJHQ0RSEkUROWVaOKuViA/3kTqcUREE\nATOTg/Hpj0fQ1tmLCyaFQO2klDosIqIxiUns6FktW9LpdHjiiScwY8aMwY/9Mjl9+OGHsXTpUgBA\nZGQk1q1bZ61Q7IZuIInVMImVSkyoJwQAZVWtUodCRBKra9ahoaUbU+IDHGLQ3vSkYGzYcQSiCMxM\nDpE6HCKiMUujVqGpnSsdR8Nqv5XVajXeeOMNBAYGnvJYeXk5Ojo6kJqaaq0vb5e6e/qSWBdWYiXj\nqnHCuAA3lNe2w2Q2Sx0OEUkop7+VOM3OW4kH+Hg44/yUECSEew9OYyciItvTOPdVYkVRlDoUu2W1\nbEmlUkGlOv3Lv/fee1ixYsXgf2u1Wtxzzz1oaGjA8uXLcdVVV1krLFnTsZ1YFmJCvVDd2IXjDZ2I\nCPaUOhwikkhOmRaCAKRGO87Rgpsvmyh1CEREY55GrYQoAr0GM5zVPNoxEjbPlnp7e5GZmYnVq1cD\nALy9vXHvvffiqquuQkdHB5YuXYrp06eftoL7SwEBHjaI1rZUFc0AgEB/d4f8/uzF5IlB2JFTg7pW\nPaamSPv3wOuAxgI5XudtnXocqW7DxAhfRE1wnCSWpCPH65zI0nidD42XhwYA4OahgY+nRuJo7JPN\nk9gDBw6c1Ebs7u6OJUuWAAB8fX2RnJyM8vLycyaxjY0dVo1TCvWNnQAAk8HokN+fvQj07JsOnVNc\nj+kJAZLFERDgweuAHJ5cr/PdebUwi0BShI8s4yP7ItfrnMiSeJ0PnaK/jbi6tg1GPdc6nsnZborY\nfFJFXl4eEhISBv973759eOqppwD0DYMqKipCZGSkrcOShYF2Yp6JlVagtws8XZ1QWtUmdShEJJGc\nMsc6D0tERPKhUfe91+eE4pGzWraUn5+PZ555BtXV1VCpVNi6dStefvllNDY2Ijw8fPB5GRkZ2LRp\nE5YtWwaTyYRVq1YhKCjIWmHJGs/EyoMgCIgJ80ZWSSOa23vgyzYPojHFYDQhv7wZQT4uCPZ1lToc\nIiJyMJr+c7A9vUaJI7FfVsuWkpOTT7s259FHHz05AJUKTz/9tLXCsCuD04m5YkdyMaFeyCppRGlV\nG6YlMoklGkuKjrVCbzAhLdYfgiBIHQ4RETkYjXNfEtvNSuyI2f/iOwfCSqx8xPSvnyhjSzHRmMNW\nYiIisqYT7cSsxI4Uk1gZ6R48E8tR21KbEOQBlVKBsmomsURjiSiKyCnVwk2jGryZRUREZEkn2olZ\niR0pJrEyotMb4axWQqngX4vUnFQKRIR44HhDJ++SEY0hx+o70dKhR2q0H38WExGRVQwmsXomsSPF\n39AyousxspVYRmJDvWAWRVTUtEsdChHZyEAr8SS2EhMRkZWwnXj0mMTKSLeeSaycxIT2tRKWsqWY\naMzILG6ESikgJcpP6lCIiMhBsZ149JjEyoQoiujWm7gjVkaiOdyJaEypb9GhqrETiRG+/FlMRERW\nwxU7o8ckVib0BhPMoghXrteRDU9XNYJ8XHCkpg1mUZQ6HCKysqziRgDAlPgAiSMhIiJHdqKdmJXY\nkWISKxO6Hq7XkaOYMC90602oaeySOhQisrKDxY1QCALSY5nEEhGR9bCdePSYxMrEifU6TGLlJDbM\nGwDPxRI5uub2HlTUtiNhgjfcXZykDoeIiByY8+B0YrYTjxSTWJnQ9V/EbCeWl+hQnoslGgsyB1uJ\nAyWOhIiIHJ1CEOCsVrISOwpMYmWClVh5CvFzhZtGhbLqVqlDISIryixugABgcixX6xARkfW5MIkd\nFSaxMsEzsfKkEAREh3qhsbUHbZ16qcMhIito69SjtKoNsWFe8HJ3ljocIiIaAzRqFacTjwKTWJlg\nJVa+BvbFlvFcLJFDyirVQgRbiYmIyHY0rMSOCpNYmeCZWPmK7d8XW8pzsUQOKbO4AQBX6xARke1o\n1Er0Gs0wmc1Sh2KXmMTKxEA7MSux8hMR4gmlQsARVmKJHE5ntwFFR1sRGeIJX0+N1OEQEdEYMbAr\nVs9q7IgwiZWJgXZinomVH2cnJcKD3FFZ14FeA3/QEDmS7NJGmEURGazCEhGRDWmcuSt2NJjEyoSO\nZ2JlLSbUGyaziMq6DqlDISILOrFah0ksERHZzkAltptJ7IgwiZUJnomVt5gwDncicjTdeiMKK5sx\nPtAdgT6uUodDRERjiEbdX4nVc0LxSDCJlYnuHiOUCgFqFf9K5GhwQjGHOxE5jNwyLYwmkVVYIiKy\nucEklpXYEWHGJBM6vREuzioIgiB1KHQaPh7O8PPUoKy6DaIoSh0OEVnAiVZirtYhIiLbGmgn5q7Y\nkWESKxM6vZFDnWQuNswLnd0G1DXrpA6FiEZJ32tCXnkTQvxcEervJnU4REQ0xrASOzpMYmWiW2+E\nC8/DytrguVi2FBPZvbzyJvQazWwlJiIiSTCJHR0msTJgNJnRazCzEitzg+diOdyJyO5llvS3Esex\nlZiIiGyP7cSjwyRWBrgj1j6EBbjDWa1kEktk5wxGM3LLtPD30iA8yF3qcIiIaAxiJXZ0mMTKwOCO\nWLYTy5pCISB6nCdqm3To7DZIHQ4RjVBBZTN6ek3IiA/kMD0iIpLEiRU7TGJHgkmsDLASaz/YUkxk\n/zKLGwCA52GJiEgyLs5sJx4NJrEyoOthEmsvYsO8AXC4E5G9MprMyCnVwsfDGZHjPKUOh4iIxii2\nE48Ok1gZGKjEujCJlb2ocZ4QBFZiiexV8bFWdPUYMTkuAAq2EhMRkUQ42Gl0mMTKwGAllmdiZc/F\nWYWwAHdU1LbDaDJLHQ4RDdNAK3EGW4mJiEhCTioFlAqBldgRYhIrAzpWYu1KTKgXDEYzjtZ3SB0K\nEQ2D2Swiq6QRnq5Og0cDiIiIpKJRK5nEjhCTWBngYCf7EhPWN9zpCM/FEtmV0qpWtOsMSI8LgELB\nVmIiIpKWRq1iO/EIMYmVgYF2YlZi7UNs/4TiUp6LJbIrmcWNADiVmIiI5EHjrEQ3V+yMCJNYGRis\nxPJMrF3w89LAy12Nsqo2iKIodThENARmUURmSSPcNCokhPtIHQ4REdFgOzHfTw4fk1gZ4JlY+yII\nAmJDvdDW1QttW4/U4RDREFTUtqOlQ4+0GH+olPzVR0RE0tOoVTCLIgxGDgsdLv4ml4ETK3aUEkdC\nQxXT31LMfbFE9uFEK3GgxJEQERH14a7YkWMSKwO6HiOc1UooFfzrsBcx/ZNNuS+WSP5EUURmcQOc\n1UokRbKVmIiI5OFEEsvhTsPFrEkGdHojJxPbmfAgd6hVCpSyEkske8cbOtHY2oNJ0X5wUrHjhYiI\n5EGj7nv/z0rs8DGJlYFuvZFDneyMSqlARIgnqhs7B6dLE5E8HexvJc5gKzEREckI24lHjkmsxERR\nhE5v5FAnOxQb5gURQHktq7FEcpZZ3AC1SoGUKD+pQyEiIhrEduKRYxIrsb6x2mA7sR2K5nAnItmr\n0XahtkmH5Cg/OKvZSkxERPIxUMRiJXb4mMRKbHBHLJNYuzM4oZjDnYhkK7O4AQAwJT5A4kiIiIhO\nxnbikWMSK7HBHbE8E2t33F2cEOLniiM17TCZud+LSI4yixuhVAiYFO0vdShEREQnGRzspGc78XAx\niZUYK7H2LSbUC/peE6oauqQOhYh+paG1G8caOpEU6cvheUREJDusxI6cVZPYkpISLFiwAO+//z4A\n4KGHHsKVV16JlStXYuXKlfjxxx8BAJs3b8aSJUuwdOlSfPrpp9YMSXYGJtsyibVPMWFsKSaSq8FW\n4ji2EhMRkfxwxc7IWS1z0ul0eOKJJzBjxoyTPv6nP/0Jc+fOPel5r7zyCjZs2AAnJyf85je/wcKF\nC+Ht7W2t0GRloBLL6cT26ZfnYudPCZM4GiL6pcziRigEAelMYomISIY4nXjkrFaJVavVeOONNxAY\nePa9fLm5uUhJSYGHhwc0Gg0mT56MrKwsa4UlOwNnYtnqZp+CfV3h7uKEsqpWqUMhol9obu9BeU07\n4sO94e7iJHU4REREpxhIYrtZiR02q2VOKpUKKtWpL//+++9j7dq18PPzw6OPPgqtVgtfX9/Bx319\nfdHY2HjO1w8I8LBovFIRlH0Xb3Cgh8N8T2NNUpQf9hfUQXBSwd/bxaKvzWuCxgJrXOd7i/paiedk\njOe/I5IFXoc0FvA6Hx5Xdw0AwAz+2Q2XTct/V199Nby9vTFx4kS8/vrrWLNmDdLT0096jiiKQ3qt\nxsYOa4Roc9rmvoFAhh6Dw3xPY834ADfsB7D/UDXOmxhksdcNCPDgNUEOz1rX+Y7MKggAYkP474ik\nx5/nNBbwOh8+s7kv72nv0PPP7jTOltjbdDrxjBkzMHHiRADAvHnzUFJSgsDAQGi12sHnNDQ0nLMF\n2ZHoeCbW7nFfLJG8tHX1ovR4K6LDvODt7ix1OERERKelUAhQOyk42GkEbJrE3n333Th+/DgAYP/+\n/YiNjcWkSZOQl5eH9vZ2dHV1ISsrCxkZGbYMS1LdPBNr9yKCPaBUCCirYhJLJAfZJY0QAWRwoBMR\nEcmcRq3iYKcRsFrmlJ+fj2eeeQbV1dVQqVTYunUrVqxYgfvuuw8uLi5wdXXFU089BY1Gg/vvvx+3\n3norBEHAnXfeCQ+PsdMTPrBih5VY+6V2UiIi2AMVtR3Q95rg3H9In4ikMbBaZ3I8k1giIpI3jVrJ\nSuwIWC1zSk5Oxrp16075+MUXX3zKxy655BJccskl1gpF1rr1RigVAtQqmxbFycJiwrxwpKYdFbXt\nSJjgI3U4RGNWZ7cBRcdaERHsAX8vyw5aIyIisjSNWom2zl6pw7A7zJwkptMb4apRQRAEqUOhURg4\nF1vKc7FEksop1cJkFjGFVVgiIrIDGrUKeoMJ5iEOt6U+TGIlptMb2UrsAAaHO/FcLJGkBlqJM+LH\nzoBAIiKyXwO7YvVsKR4WJrES6+4xwpVJrN3zcndGgLcGR6rbeCeNSCLdeiMKKpsRFuCGIF9XqcMh\nIiI6p4FiFs/FDg+TWAkZTWb0Gs2sxDqImFBv6PRG1Gq7pA6FaEzKPaKF0SRiCquwRERkJwYqsZxQ\nPDxMYiWk43odhxI3vq+lOK+8WeJIiMamzOJGAOB5WCIishsnklhWYoeDSayEBnbEshLrGCbHBUCp\nELAnv07qUIjGHL3BhLzyJgT5uiLU303qcIiIiIZEo+5vJ9azEjscTGIlNLAjlmdiHYOHqxqp0X6o\nauzEsfoOqcMhGlPyy5vQazAjIz6A096JiMhusBI7MkxiJTRQiWUS6zhmpYQAAKuxRDbGVmIiIrJH\nTGJHhkmshAYqsS48E+swUqP94O7ihH0FdTCazFKHQzQmGIxm5B7Rwt9LgwlBHlKHQ0RENGQD7cTd\nHOw0LExiJaRjJdbhqJQKTJsYhHadAfkVHPBEZAvfHTyObr0JGQmBbCUmIiK7wkrsyDCJlRDbiR3T\nzJRgAGwpJrKF2qYubNpZAU83NS6bPkHqcIiIiIaFK3ZGhkmshAbbiZnEOpSIYA+M83dDTmkjunoM\nUodD5LDMooh3vimC0WTGioVxcHdxkjokIiKiYTkxnZiV2OFgEiuhbu6JdUiCIGBmcjCMJhE/H26Q\nOhwih7U9qxqlVW2YEh+AjIRAqcMhIiIaNo0z24lHgkmshHTcE+uwZiQFQxCAPfm1UodC5JC0rd3Y\n8OMRuGlUWLEwTupwiEOUFv8AACAASURBVIiIRmSwEst24mFhEishVmIdl4+HMxIjfHGkuh11zTqp\nwyFyKKIo4t1vi6A3mHD9glh4uTtLHRIREdGIcLDTyDCJldDgmVg1k1hHNCt5YMATq7FElrQrrxYF\nlS1IifLDjKRgqcMhIiIaMbVKAUFgEjtcTGIl1K03QqNWQqHgSghHlB4XAI1aib35dTCLotThEDmE\n1k49Pv6hDBq1Er+9JJ4rdYiIyK4JggAXtYrtxMPEJFZCOr2RrcQOzNlJiYyEQDS161F8rFXqcIjs\nniiKWLe1GDq9EUvnxsDXUyN1SERERKOmcVayEjtMTGIl1K03cqiTgxtsKc5jSzHRaB0oakB2qRbx\n471xYdo4qcMhIiKyCI1axSR2mJjESkQUxb5KLJNYhxY73hv+XhocLG5kmwjRKHToevHBdyVQqxS4\n6bIEKNhGTEREDkKjVvJ94jAxiZVIT68Josj1Oo5O0b8zVm8wIaukUepwiOzWR9+XokNnwKLZUQjy\ncZU6HCIiIovRqJUwmkQYTWapQ7EbTGIlwvU6Y8fM/pbi3Xl1EkdCZJ9yyrTYV1iPyBBPXDR1vNTh\nEBERWdSJXbFsKR4qJrES0fUnsazEOr5AH1fEhHmh6GgLmtt7pA6HyK7oeoxYt7UYSoWAWy5L4DR3\nIiJyOAO7YgeKXHRuTGIlMrAjlmdix4ZZycEQAewtYDWWaDg+2V6Glg49rpwZgdAAd6nDISIisriB\nJJaV2KFjEiuRwXZiJrFjwtSEIDipFNidVweRO2OJhuRwZTN+yq1BWIAbLpsxQepwiIiIrOJEOzEr\nsUPFJFYig+3EPBM7JrhqVEiP9Uddsw7lte1Sh0Mke/peE9Z+UwRBAG6+bCJUSv66IiIix8RK7PDx\nXYFE2E489sxMDgEA7MlnSzHRuXz+Uzm0bT245LxwRIZ4Sh0OERGR1TCJHT4msRJhO/HYkxTpAy83\nNX4urIfByBHqRGdSVt2G7w8eR5CPC64+P1LqcIiIiKxqsJ2Yg52GjEmsRDideOxRKhSYkRSMrh4j\ncsu0UodDJEsGoxlrtxyGiL42YrWTUuqQiIiIrIqV2OFjEisR7okdmwZ2xrKlmOj0vtxTgdomHeZN\nDkXceG+pwyEiIrI6jfNAEstK7FAxiZUIz8SOTWGB7ggPckdeeRPau3qlDodIVo7Vd2DL3mPw83TG\nkgujpQ6HiIjIJk5MJ2YldqiYxEqkm+3EY9as5BCYzCL2F9ZLHQqRbBhNZry95TDMoojfXprAn41E\nRDRmuLCdeNiYxEpEpzdCpRTgpOJfwVgzLTEISoWA3f/P3p2HR1Wf/eN/n9mz7wlZyEIIawiENaDI\nLogFwSItKKIPXSwutd9Wni5q+9Rej1Xb/vpYoVpqUZG2ClZFBEE2RfY1kIgkIRCykj2TZPaZ8/sj\nyYQlyWSZOWcyeb+ui6vJMJlzZzzQvLnvz+eTUy53KUReY/eJa7h2vQl3jBmE9JQIucshIiKSDM+J\n7TkmKJkYzTb4aVUQBEHuUkhiwQEajBkSgWvXm1BS2SR3OUSyK6tuxsdfXUVIgAbfnZMmdzlERESS\nal8Ty05sdzHEysRgsnE97ADGDZ6IWtgdIjbtugib3YFV84cjQKeWuyQiIiJJcXfinmOIlUlbJ5YG\nprFDIxGgU+FobgXsDp4ZSwPXp18V4nKpHpNGRGP8sCi5yyEiIpKcUqGAWqXgOHEPMEXJwGZ3wGJz\n8HidAUytUmDyyBgcOFuKr6/WYcwQrgEk71arNyGvpB75xQ0ormyCUiFAo1ZCq1ZAq1ZCo1FCq1JC\no1ZAq1FCo1JCq1ZCq2l5jkbV8rFGrYRWpYBGo0Sz0Yp3dl1EoJ8aD84bJve3SEREJBudRgmjmZ3Y\n7nKZohoaGlBZWYm0tDQcOnQI58+fx/LlyxEVxX8x7y0DdyYmANPGDMKBs6U4fKGcIZa8ikMUUV5j\nQH5xvTO41uhNzt8XBEAU3Xe9hxcNR3CAxn0vSERE1M/oNEp2YnvAZYp65plnsHr1aqjVavz+97/H\nypUr8atf/Qp/+9vfpKjPJxl5RiwBGBIbjEHh/jibX92yRtoDnfmcKzXYtPMbNJusUCsVULX9Uimg\nViqgVgnOx9Sqtt9v2TVbfcNzVUoF1EoBkaF+yBoVww3JJHKhsAZfnCtDWKAWESE6RIXqEBnih4gQ\nHQJ07tsYzmZ3oKii0RlY80vq0Wxq/z/SQD81MtMikZYQirTBIUiKCYJCIcBitcNidcBstTt/tX1u\nueVzs8UOi80Os9Vxw8d2DE8KR9aoGLd8H0RERP2VTqNCo8Eodxn9hsufmo1GI+644w68/vrreOih\nh7BixQrs3btXitp8FjuxBACCIGBa+iD858tCnPzmOmaMi3fr6x84U4Itn+dDoRAQHxUAm90Bm80B\nm90Bo9kGfevHNpsDPWmqhQZqMTIpzK210u3Ka5qx4aMcmDvZ5EGnUSIypD3URjp/+SEyVAf/LnY/\nN5ptKCzTI6+4Hvkl9Sgs08Nia1+bHRmiQ0ZqBNIGh2JYQigGRfhD0cFr6TQq6PrYQI2KCkJVVWPf\nXoSIiKif02mUMFvsEEWRzYJu6FaIra2txe7du7FhwwaIooiGhgYpavNZbSGWa2JpWvogfPhlIY7k\nVLgtxDocIv69Px97T5UgyF+NJ7+dgaHxIZ0+XxRF2B1iS6C1i7DaHLDeEHjbPq6oNeDtzy7hwJkS\nhlgPM1lsWP9hS4B9dOEIJEYHobrBiOoGE6obTKhpMKG6wYiqBhNKqpo7fA0/rRIRwX7OcBserENt\nown5JQ0ovt4ER+s8sAAgPioQaYNDMCwhFGkJIQgP1kn43RIREZFOo4IIwGy1O8+Npc65fIcWLVqE\nu+++Gw888ABiY2Px2muvYcqUKVLU5rPaxonZiaXwYB1GJIXhYlEdKusMiA7z79PrGc02/G17LrIv\n1yAuMgA/XpaBqFC/Lr9GEASolC1jxV0ZNjgU+8+U4kxeNeoazQgL0vapVuqYKIp4+7NLKKtuxtwJ\nCZieEQcASBoU1OFzm002Z6itbjChur71Y70JVQ1GlFTdfBaxSilgSHywM7AOTQjhsTZEREQyu/GY\nHYZY11y+Q6tXr8bq1atv+jwo6PYfpjqSl5eHtWvX4pFHHsFDDz2E8vJy/OIXv4DNZoNKpcIrr7yC\nqKgojB49GuPHj3d+3VtvvQWlUtmLb6d/cHZiGWIJLd3Yi0V1OJJTgSXTh/T6dWr1Jvx563mUVDVh\ndEo4fnRfulu7/YIgYNb4eLzz2SV8mV2G++5McdtrU7v9Z0px/OvrSI0PxvLZQ7t8riAICPRTI9BP\n3WXIrW4woqbBhEA/NYbEBUOt8t2/X4mIiPojnhXbMy5/wj127Bg2b96MhoYGiDdsR7lly5Yuv85g\nMOCFF17A1KlTnY/9+c9/xvLly7Fw4UJs2bIFmzZtwrp16xAYGIjNmzf34dvoXwzc2IluMGF4FN7d\nk4cjORVYfGdKh2sPXblSrser286jodmCWePjsXJuGpQK9x8DnTUqBlsPFOCLc6W4d2qSy+4t9UxB\naQP+vS8fQf5q/Oi+9D6/vzeG3ORBwW6qkoiIiNytrfvKHYq7x2WK+vWvf40f/ehHiIuL69ELazQa\nbNy4ERs3brzptbTalhHEsLAw5Obm9rBc32Dkmli6gU6jwsThUTicU4H84noMT+zZetNT31Ti7zu+\nhtXuwIq5aZg7IcFjGwLoNCpMS4/FvtMlOJdfjYkjoj1ynYFIb7Dgrx/lwCGK+OHi0VyXSkRENIA4\nO7E8K7ZbXKaohIQELFmypOcvrFJBpbr55f39W9b72e12/POf/8Tjjz8OALBYLPjpT3+K0tJSzJ8/\nH48++miPr9efcHdiutW09EE4nFOBwzkV3Q6xoihi57EifPBFIbQaJZ5akoGxQyM9XCkwe3w89p0u\nwf4zJQyxbuJwiHjj41zUNZrx7RlDMCo5XO6SiIiISEJtuYDjxN3jMkVNnz4d7733HiZPnnxTKB08\neHCvLmi327Fu3TpkZWU5R43XrVuHxYsXQxAEPPTQQ5g4cSLGjBnT5etERXVvXa43Elu7ZAlxoYgK\n79tGPuQbIiIC8dbuSzh9qQo/XjHeOVLS2X1utTnw2tZz2H+qGJGhfnh+zRSkxHW+A7E7RUUFIWNo\nJM4XVMPkAAbH9N8/i95i866LuFhUh8mjBuHhb6VDoRhYW+v357/PibqL9zkNBLzPey8yIgAAoNap\n+T52g8sQ+8477wAA3njjDedjgiBg3759vbrgL37xCyQlJeGJJ55wPrZixQrnx1lZWcjLy3MZYvvz\nuYK19S0HGZuaTaiy819bqMWUkdHYcaQInx+5gqzRgzo9P7PJaMVr/7mAvOJ6pMQG4clvZyBQrZD0\nz8Sd6YNwvqAaH+zLw4Pzhkl2XV90rqAa7+/NQ1SoDqvuTkNNTZPrL/IhPCeWBgLe5zQQ8D7vG5vZ\nCgCoqm7i+9iqqzDvMsT+61//QkxMjFsK2b59O9RqNZ566innY4WFhVi/fj3+8Ic/wG6348yZM1iw\nYIFbruet2tbEcvtsutHU0YOw40gRDudUIGv0oA6fU1FrwJ+3ZqOyzoiJw6Ow5lujoFVLv9PsuLRI\nhAZqcCSnHN+eMYT3ci9V1hvx90++hlqlwONLx/CoGyIiogGqfWMnNri6w+VPns8884yzG9sTOTk5\neOmll1BaWgqVSoXdu3ejpqYGWq0Wq1atAgCkpqbiN7/5DQYNGoRly5ZBoVBg9uzZyMjI6Pl30o8Y\nzDb4aZUDbmSQuhYbEYDUuGB8fbUWdY3m2/716WJRHTZ8eAHNJhvunZqEpXcN6dVOxu6gUiowY1w8\nPv7qCo59fR0zx8XLUkd/ZrHaseHDCzCYbXh04QgkciybiIhowGo/Yoe7E3eHyxCbnJyMdevWITMz\nE2p1e5dg2bJlXX5denp6t4/NeeaZZ7r1PF9hNNu4qRN1aNqYWFwu0+NYbgWGDWnfpOlQdhne2X0J\nAPBfC0fizoxYuUp0umtsHD45fBX7T5dixtg4j+2I7Ku2fJ6Ha9ebcNfYWEzP6Nnu70RERORbdNqW\nEGvk7sTd4vIQQqvVCqVSifPnz+P06dPOX9R7BpONZ8RShyaPjIZKKeBwTgVEUYRDFLH1QAE27foG\nOo0SP/vuOK8IsAAQFqTF+GGRKKlqwuVSvdzl9CtfZpfh0PlyJMUEcU0xERER8ZzYHnKZpF588UUp\n6hgwHKIIo8UGP22A3KWQFwrQqTFuaCROXapCbmENtu7Nw5m8KsSE++PpZRmI8bLdrGeNT8CpS1XY\nf7YEQxOk2R25vyuqaMS7e/Lgr1Vh7dJ0qFXSr2kmIiIi79I+TsxObHe4DLEzZszocEzw4MGDnqjH\n55ktdogi2ImlTk0bE4tTl6rw3BtHYbM7MCIxFGuXjkGgn/dt+jMiMRSxEf449U0lvjs7DcEBGrlL\n8mrNJivWf3gBNrsDjy9NR1Son9wlERERkRdgiO0Zl0nqn//8p/Njq9WKo0ePwmQyebQoX9a2M7Gf\njiGWOpaeEo5gfzX0BivuzIjFw/OHQ6V0OfkvC0EQMHt8ArZ8nodD58tw79RkuUvyWg5RxN8/+RrV\nDSYsmpaMsUMjXX8RERERDQhatRICOE7cXS6TVHz8zbuOJicnY82aNXj00Uc9VpQvM5habkx2Yqkz\nKqUCT9yfAZsgYHhckNdvmDR19CBsO3gZB8+W4p4pSdx1uxM7jxYh+3INRieH4b47U+Quh4iIiLyI\nIAjQapTsxHaTyyR19OjRmz6vqKjAtWvXPFaQrzO0dWIZYqkLQxNC+s2h4f46FaaOjsHBc2U4f7kG\n49LYYbxV7tVafHioEOHBWvxg8WgGfSIiIrqNTqNkJ7abXCapDRs2OD8WBAGBgYH4n//5H48W5cva\nQqw/x4nJh8zMjMfBc2XYf7akX4ZYfbMFm3ZeRGl1M0YmhWHMkAiMSg53y5/TWr0Jb3ycC4Ug4EdL\n0hHkz3XDREREdDudRoVmk1XuMvoFlz+hPf7448jKyrrpsb1793qsIF9nNLETS74nMSYIQxNCkFNY\ni8o6A6LDvGsX5a4Ulumx/sMLqGs0Q6NW4ND5chw6Xw6lQkBqfAjGDAnHmCERGBwd2OPRbpvdgQ0f\n5aDJaMVDdw9Dahx3cCYiIqKO6TRK1Oi591B3dJqkSkpKUFxcjJdeegk///nPIYoiAMBms+F///d/\nMXfuXMmK9CXOTixDLPmY2ZnxKChpwMGzZVg+e6jc5XTLl9lleHfPJdgdIr49YwgWTEnE1YpGXLhc\ngwuFtcgvrkdecT0++KIQoYEapA+JQEYPurTv7StAYZkeWaNjMCsz3uXziYiIaODSaZSw2hywOxxQ\nKrxzU09v0elPYVVVVdi5cydKS0uxfv165+MKhQLf/e53JSnOF3GcmHzVhOHRCNqXj0Pny7Bkego0\nau89/9Rqc+Bfe/Nw8FwZAnQq/PC+0UhPiQAApMaFIDUuBEumD4HeYEHulVpcKKxBTmEtvjpfjq/O\nl0MhCBgaH4wxqRGddmmP5VZg35kSxEcGYPX8EV6/QRcRERHJq21S02SxI0DHENuVTpNUZmYmMjMz\nMWPGDHZd3cjIjZ3IR6lVCtw1Ng6fHi3CiYuVuDMjVu6SOlTXaMaGDy/gcpkeg6MD8cT9Yzo9rzXY\nX4Opowdh6uhBcIgirpY34kJhDS4U1iC/pAF5JQ344ItChARqMCYlAmNSIzA6OQx1jWa89dk30GmU\nWLs0HVqN9wZ6IiIi8g7Os2LNdgTo1DJX491cJqkRI0bgqaeeQl1dHTZv3oytW7di0qRJSE5OlqA8\n38MjdsiXzRgXh53HinDgbIlXhti84nps+CgH+mYLskbHYPWCEdB2s2OsEAQMiQvGkLhg3HdnCpqM\nVuRcqcGFy7XIuVKDry6U46sLLV1arUYJi9WBtUvSERsR4OHvioiIiHyBTtPWieUOxa64TFLPP/88\nHnzwQWzatAlAyzmxzz33HDZv3uzx4nyRkWtiyYdFhvhhbGokzhVU40q5HimxwXKXBAAQRRH7Tpfg\nvf0FEEVgxZw0zJ2Y0KcR30A/NbJGDULWqJYubVFFe5e2sEyPhVlJmDgi2o3fBREREfkyZyeWZ8W6\n5DJJWa1WzJkzB2+99RYAYNKkSZ6uyadxTSz5utnj43GuoBoHzpQi5V75Q6zFasfbn13C0dwKBPur\n8aMl6RieGObWaygEASmxwUiJDcbiO1JgszugUnItCxEREXUfQ2z3deunLL1e7+xY5Ofnw2w2e7Qo\nX2Y026BSClCruEaOfNOolHBEh/rh+MXraDLKe9ZZdb0R//vuaRzNrUBKbDCef2SS2wNsRxhgiYiI\nqKfaxonbJjepc906J3b58uWoqqrCokWLUFdXh1deeUWK2nySwWTjKDH5NIUgYGZmPN4/UIDDF8ox\nf3KiLHXkXq3FGx/nosloxV1jY/HgvOFQqxguiYiIyDuxE9t9LtPUlClT8NFHHyEvLw8ajQYpKSnQ\narVS1OaTjGYbdyYmn3dnRiw+PFSIA2dLMW/SYCgkPF5GFEV8dvwatn1xGUqFgNULhmPGOJ7RSkRE\nRN5Np+XGTt3lsi3x8MMPQ6fTISMjAyNGjGCA7SOD2cb1sOTzAv3UmDwyGpV1Rnx9tVay65osNvz1\n41xsPXgZoYFa/PfK8QywRERE1C+wE9t9LtPUyJEj8X//93/IzMyEWt1+XtHUqVM9WpgvstocsNoc\n7MTSgDB7fAIOX6jA/tOlSE+J8Pj1rtca8Jf/XEBZdTOGJYTgR0vHICRA4/HrEhEREbkDQ2z3uUxT\nFy9eBACcOnXK+ZggCAyxvcDjdWggSYkNRvKgIGRfrkZ1gxGRIX4eu9a5gmps/CQXRrMdcyckYPns\nodxciYiIiPoVnhPbfS7TVFfnwW7cuBHf//733VqQL2sLsezE0kAxa3w8Nu38Bl+cK8O3Z6S6/fUd\noojtX13B9sNXoVYp8P1vjcLU9EFuvw4RERGRp7ET2319alUcOnTIXXUMCDwjlgaaySNjEKBT4VB2\nGaw2h1tfu6SyCS9uPo3th68iMkSHXz40gQGWiIiI+i2G2O7rU5oSRdFddQwIBhM7sTSwaNVK3JkR\ni90ninE6rxJZo/oeMs1WO7YfvoI9J4phd4iYNCIaq+YPR6Cf2vUXExEREXkpjhN3X5/SlCDhsRm+\ngGtiaSCamRmP3SeKceBMaZ9DbE5hDd7ZfQnVDSZEBOuwav5wZKR6ftMoIiIiIk9TqxRQKQV2YruB\naUpCBq6JpQEoJswf6SnhyLlSi+LKJgyODuzxazQ0mfHv/QU4/vV1KAQB90xJxOI7UqBtHbshIiIi\n8gU6jYohthuYpiTUNk7MNbE00MwaH4+cK7U4cLYUD88f3u2vc4givswuw7YDl2Ew2zAkLhgPzx+O\nxJggD1ZLREREJA+dRslx4m7oU5pKTk52UxkDg4HjxDRAjU2NRESwFkdzKvDAzNRuTSOUVjXh7d2X\nUFDSAD+tEg/dPQwzx8VDoeAyBiIiIvJNOo0StXqz3GV4PZe7E5eWluKpp57CqlWrAADvv/8+rl69\nCgD47W9/69HifA2P2KGBSqEQMGNcPMxWO47kVHT5XIvVjg++uIzfbDqJgpIGTBwehd99Lwuzxycw\nwBIREZFPaxsn5ga6XXMZYp977jncd999zjcyJSUFzz33nMcL80UcJ6aBbPrYOCgVAvafKen0L+bc\nq7V4/s0T+PRoEUIDNXhqWQbWLh2DsCCtxNUSERERSU+nUcIhirC4+WhCX+MyxFqtVsyZM8e5E/Gk\nSZM8XpSv4u7ENJCFBGgwcUQ0ymsMuHSt/qbf0zdbsPGTXPzx3+dQ1WDE/MmD8cL3pmDc0EiZqiUi\nIiKSHs+K7Z5upSm9Xu8Msfn5+TCbOafdGwazDQIAHUMsDVCzMuNx/Ovr2H+2FCOSwiCKIg6dL8fW\nAwVoNtmQPCgIqxeMQNIgbtxEREREA8+NZ8WGBGhkrsZ7uUxTjz/+OJYvX46qqiosWrQIdXV1eOWV\nV6SozecYzTbotEooeL4uDVBpCSFIiArE2bwqXLxai48PX0VecT20GiVWzk3julciIiIa0JydWDM7\nsV1xGWKzsrLw0UcfIS8vDxqNBikpKdBquT6tNwwmG0eJaUATBAGzx8fjnd2X8Mq/zwEAxg+Lwsq5\naQgP1slcHREREZG8dNq2cWIes9OVThPVa6+91uUXPvHEE24vxtcZzTaEB/MfAGhgyxodg48PX4FC\nEPDQvGHIHBYld0lEREREXqF9nJid2K50GmJttpb0X1RUhKKiIkycOBEOhwMnTpzAqFGjJCvQVzhE\nEUazDf7aALlLIZKVTqPC/34/C2qVAiqly73liIiIiAYMbuzUPZ2G2KeffhoA8Nhjj2Hr1q1QKlve\nUKvVip/85CfSVOdDzBY7RPCMWCKAfw6IiIiIOtIeYjlO3BWXbZDy8vKbznQUBAFlZWUeLcoX8YxY\nIiIiIiLqCseJu8dlopo5cybmz5+P0aNHQxAEXLx4EXPmzJGiNp9iaD0jlh0oIiIiIiLqCMeJu8dl\novrJT36CpUuXIi8vD6Io4sknn8TQoUOlqM2nGM3sxBIRERERUefaGl4cJ+6ay0Rlt9tx7tw55OTk\nAGhZE8sQ23Nt48TsxBIRERERUUfYie0el4nqhRdeQG1tLaZMmQJRFLFr1y6cO3cOzz77rBT1+Qxn\nJ5YhloiIiIiIOsA1sd3jMlEVFBTg3XffdX7+0EMPYeXKlR4tyhcZnOPEapkrISIiIiIib+TsxJo5\nTtwVl7sTW61WOBwO5+d2ux12O/9loKfaN3ZSylwJERERERF5I21riDWyE9sll53YGTNmYNmyZZg0\naRIA4Pjx41i4cGG3XjwvLw9r167FI488goceegjl5eVYt24d7HY7oqKi8Morr0Cj0WD79u14++23\noVAosHz5cjzwwAN9+668kLHtiB0tO7FERERERHQ7hSBAq1ZyYycXXIbYtWvXYtq0acjOzoYgCPjt\nb3+LjIwMly9sMBjwwgsvYOrUqc7HXn31VaxcuRL33HMP/vSnP2Hbtm1YsmQJ1q9fj23btkGtVmPZ\nsmWYN28eQkND+/adeRl2YomIiIiIyBWdRsk1sS64HCduaGhAQEAAVq9ejeTkZBw6dAhVVVUuX1ij\n0WDjxo2Ijo52Pnb8+HHnGbOzZs3C0aNHkZ2djTFjxiAoKAg6nQ7jx4/HmTNn+vAteSeuiSUiIiIi\nIlcYYl1zGWKfeeYZVFZW4urVq3j55ZcRGhqKX/3qVy5fWKVSQafT3fSY0WiERqMBAERERKCqqgrV\n1dUIDw93Pic8PLxbIbm/ad+dmJ1YIiIiIiLqmE6j4jixCy7HiY1GI+644w68/vrrePDBB7FixQrs\n3bu3zxcWRbFHj98qKiqozzVIyWp3QK1SIC7Wt8akybP6231O1Bu8z2kg4H1OAwHvc/cICtSg6Hoj\nwiMCoVQIcpfjlboVYmtra7F7925s2LABoiiioaGhVxfz9/eHyWSCTqfD9evXER0djejoaFRXVzuf\nU1lZiXHjxrl8raqqxl7VIBd9kwV+GmW/q5vkExUVxPuFfB7vcxoIeJ/TQMD73H3a5jZLSuvhr3MZ\n13xWV/8o4nKceNGiRbj77ruRlZWF2NhYrF+/HlOmTOlVIdOmTcPu3bsBAHv27MH06dMxduxYXLhw\nAXq9Hs3NzThz5gwmTpzYq9f3ZgazDX5cD0tERERERF3QaVuCK0eKO+cy2q9evRqrV6++6fOgINej\nAjk5OXjppZdQWloKlUqF3bt34w9/+AN+/vOf47333kNcXByWLFkCtVqNn/70p1izZg0EQcDjjz/e\nrdfvb4xmGyKCtXKXQUREREREXkzXelYsN3fqXKch9ne/+x2effZZrFy5EoJw+yz2li1bunzh9PR0\nbN68+bbHN23ajSToRAAAIABJREFUdNtjCxYswIIFC7pTb79ktTlgtTngrx244wBEREREROQaQ6xr\nnaaqZcuWAQCefvppyYrxVUbnGbEMsURERERE1Dk/DceJXel0TeyIESMAABMmTEBzczOys7Nx/vx5\nmM1mTJo0SbICfUH7GbEMsURERERE1Dl2Yl1zubHTL3/5S7z55pvQ6/Wor6/HX//6Vzz33HNS1OYz\nDCZ2YomIiIiIyDVu7OSay1R1+fJlbNu2zfm5KIpYvny5R4vyNW3jxFwTS0REREREXWEn1jWXndiY\nmBiYzWbn5xaLBYMHD/ZoUb6mfZyYR+wQEREREVHn2kJsWyOMbueyNSiKIubOnYvx48dDFEVkZ2cj\nLS0N69atAwC8/PLLHi+yv2vf2Enp4plERERERDSQ6ZwbO7ET2xmXIXbevHmYN2+e8/NZs2Z5tCBf\n1LYm1l/LTiwREREREXWO48SuuQyxS5cuRV5eHq5du4a5c+dCr9cjODhYitp8hoGdWCIiIiIi6ob2\nEMtx4s64DLFvvfUWduzYAYvFgrlz52LDhg0IDg7G2rVrpajPJxhNXBNLRERERESucZzYNZcbO+3Y\nsQPvv/8+QkJCAADr1q3DwYMHPV2XT2EnloiIiIiIuoPjxK65DLEBAQFQKNqfplAobvqcXGs/Yoed\nWCIiIiIi6pxapYBCEDhO3AWX48SJiYl47bXXoNfrsWfPHuzcuROpqalS1OYzDGYbBAA6dmKJiIiI\niKgLgiBAp1GyE9sFly3V559/Hn5+foiJicH27dsxduxY/PrXv5aiNp9hMNmg06qgEAS5SyEiIiIi\nIi+n0yphMjPEdsZlJ1atVmPNmjVYs2bNbb/305/+FH/84x89UpgvMZpt8GcXloiIiIiIukGnUaGh\nySx3GV6rT4tbKysr3VWHTzOYbfDjelgiIiIiIuoGjhN3rU8hVuB4rEsOUYSJnVgiIiIiIuomP40S\ndocIq80hdyleidsMe5jJbIcInhFLRERERETd035WLHco7ghDrIcZzFYAPCOWiIiIiIi6h2fFdq1P\nIVYURXfV4bOMrbuK8YxYIiIiIiLqjvZOLENsR/oUYhcuXOiuOnyWwdTaidW53AiaiIiIiIgIutYp\nTqOZ48QdcZmsduzYgY0bN0Kv10MURYiiCEEQcPDgQaxYsUKKGvu19k4sQywREREREbnGceKuuUxW\nf/nLX/C73/0OcXFxUtTjc9rWxPqzE0tERERERN3AjZ265jJZJSUlYdKkSVLU4pPaOrF+7MQSERER\nEVE3sBPbNZfJKjMzE3/6058wefJkKJXtO+xOnTrVo4X5irY1sRwnJiIiIiKi7mCI7ZrLZHXkyBEA\nwNmzZ52PCYLAENtN7MQSEREREVFPcJy4ay6T1ebNm6Wow2dxTSwREREREfUEO7Fd6zRZ/e53v8Oz\nzz6LlStXQhCE235/y5YtHi3MVxjYiSUiIiIioh5giO1ap8lq2bJlAICnn376tt/rKNRSx4zONbFK\nF88kIiIiIiLiOLErnYbYESNGAAAmT56M5uZmNDQ0AAAsFgt+9rOfYdu2bdJU2M8ZzHaolAqoVQyx\nRERERETkmq61AWYysxPbEZczrhs3bsQbb7wBi8UCf39/mM1mLFq0SIrafILBbON6WCIiIiIi6rb2\ncWJ2YjuicPWE3bt348iRIxg7diyOHTuGP/zhD0hLS5OiNp9gNNu4HpaIiIiIiLpNqVBAo1JwTWwn\nXIbYgIAAaDQaWK0tazvnzJmDffv2ebwwX2Ew2XhGLBERERER9YhOo2SI7YTLdBUSEoLt27dj2LBh\n+MUvfoHU1FRUVlZKUVu/Z7XZYbM7uKkTERERERH1iE6j4jhxJ1yG2Jdeegk1NTWYN28e3n77bVRU\nVOBPf/qTFLX1e87jdXRqmSshIiIiIqL+RKdRosFgkbsMr+QyxG7evBk/+MEPAACPPfaYxwvyJQbn\n8TocJyYiIiIiou7TaZQwW+xwiCIUPOL0Ji7XxObl5aGoqEiKWnyOsbUTyxBLREREREQ9oWvNEGau\ni72Ny3R16dIlLFy4EKGhoVCr1RBFESaTCcePH5eivn7NYG7pxPrxiB0iIiIiIuqB9mN27Dzt5BYu\n343o6Gi88cYbEEURgiBAFEXcf//9UtTW77ETS0REREREvXHzWbFaeYvxMp2mq+3bt2P9+vUoLy/H\nypUrnY/bbDbExsZKUlx/xzWxRERERETUGzpNS4bgMTu36zRdLV68GPfeey9+9atf4cknn3Q+rlAo\nEB0dLUlx/V1bJ5btfyIiIiIi6okbx4npZl2mK6VSid///vdS1eJz2tbE+nNNLBERERER9UB7J5Zn\nxd7K5e7E1HtGEzuxRERERETUc+zEdo4h1oOcnViGWCIiIiIi6gGG2M5Jmq62bt2K7du3Oz/PyclB\neno6DAYD/P39AQD//d//jfT0dCnL8hiuiSUiIiIiot7gOHHnJE1XDzzwAB544AEAwIkTJ7Br1y4U\nFBTgxRdfxLBhw6QsRRIGkxUCAJ1WKXcpRERERETUj/i1ZgiTmZ3YW8k2Trx+/XqsXbtWrstLwmC2\nQ6dVQSEIcpdCRERERET9CI/Y6Zwsc67nz59HbGwsoqKiAACvvvoq6urqkJqail/+8pfQ6XRylOV2\nRrOV62GJiIiIiKjH2tfEcpz4VrIkrG3btmHp0qUAgIcffhjDhw9HYmIifv3rX2PLli1Ys2aNy9eI\nigrydJl9ZrLYERXm3y9qJe/Ee4cGAt7nNBDwPqeBgPe5e6m0agCAQxD43t5ClhB7/PhxPPvsswCA\nefPmOR+fPXs2du7c2a3XqKpq9Eht7uIQRRhMNmhUCq+vlbxTVFQQ7x3yebzPaSDgfU4DAe9z9zO3\njhE3NJoG5HvbVXCXfE3s9evXERAQAI1GA1EU8cgjj0Cv1wNoCbdpaWlSl+QRJrMNIni8DhERERER\n9ZxGrYAgcE1sRyRPWFVVVQgPDwcACIKA5cuX45FHHoGfnx9iYmLw5JNPSl2SRxjMLbPrPF6HiIiI\niIh6ShAE6DRK7k7cAckTVnp6Ov7+9787P1+4cCEWLlwodRkeZzC1hFh/HUMsERERERH1nE6j4sZO\nHZDtiB1fZ2QnloiIiIiI+kCnUXKcuAMMsR7SNk7MNbFERERERNQbDLEdY4j1kLZOLMeJiYiIiIio\nN3QaFWx2B2x2h9yleBWGWA9xrollJ5aIiIiIiHpBp1EC4A7Ft2KI9RCuiSUiIiIior5oD7Hc3OlG\nDLEeYuA4MRERERER9YFO05Il2Im9GUOsh7ATS0REREREfaHTcpy4IwyxHsI1sURERERE1BftnViO\nE9+IIdZD2IklIiIiIqK+cK6JNbMTeyOGWA8xmG1QqxRQq/gWExERERFRz7WFWCM7sTdhwvIQg9nO\nLiwREREREfWaHzd26hBDrIcYTVauhyUiIiIiol7jObEdY4j1EIPZxuN1iIiIiIio17ixU8cYYj3A\narPDZhc5TkxERERERL3GTmzHGGI9gMfrEBERERFRX3F34o4xxHqAgcfrEBERERFRH+m0HCfuCEOs\nB7SFWK6JJSIiIiKi3uI4cccYYj3AyE4sERERERH1kUqpgEopMMTegiHWA7gmloiIiIiI3EGnUXGc\n+BYMsR7Q1olliCUiIiIior7QaZTsxN6CIdYDnBs7cU0sERERERH1AUPs7RhiPYCdWCIiIiIicged\ntmWcWBRFuUvxGgyxHsA1sURERERE5A46jRKiCFhsDrlL8RoMsR7A3YmJiIiIiMgddJq2s2I5UtyG\nIdYDnJ1YroklIiIiIqI+cJ4Va+YOxW0YYj3AaLZBAKBtveGIiIiIiIh6wxli2Yl1Yoj1AIPZBj+t\nCgpBkLsUIiIiIiLqx9rHidmJbcMQ6wFGs42jxERERERE1Gd+rZ1YIzuxTgyxHtDWiSUiIiIiIuqL\n9nFidmLbMMS6mcMhwmi283gdIiIiIiLqM+5OfDuGWDdr+xcSdmKJiIiIiKiv2ncnZohtwxDrZjxe\np3OiKMJit8hdBhERERFRv8Fx4tsxabmZwTzwOrGiKKLZZoDe3IgGsx4NFj305kbUW/TQm/VosLQ8\nrrfoYXXYsGjIAixIni132UREREREXk+n5TjxrQZO0pKIsTXE+tKa2HpzA0qbKloDqR4N5kboLfrW\nwNoIvVkPm9j5HyqFoECQOhCxAYNQZazBZ1f3ISt2AkK1IRJ+F0RERERE/Q/Pib2d7yQtL+Erndjr\nhipkV+UguyoXV/XXOnyOQlAgRBOM+KA4hGqCEawNRogmCCHaYAS3/m+INhiB6gAohJbJ9SNlJ7Dl\nm234pHA3Vo1cLuW3RERERETU7/Cc2Nv176TlhfrrmlhRFHGtsQTZVbnIrspBhaESQEtQHRY2FMNC\nUxGqbQmqoa0hNUDt7wyn3ZUVOxEHir/C8fLTmJVwJxKC4jzx7RARERER+QR2Ym/Xv5JWP9Cfxont\nDjsK6q8gu7ql41pvbgAAqBVqjI0cjYyo0UiPHIlAdYDbrqkQFFg69F6sz34THxZ8iifGfQ+CILjt\n9YmIiIiIfImWIfY23p+0+hnnOLGXdmItdgsu1uYhuyoXOdUX0WwzAAD8VH6YPGg8xkalY2T4MGiV\nGo/VMCpiOEaGD8PF2jx8XZuH0RHDPXYtIiIiIqL+TCEI0GqUHCe+gXcmrX7MGzuxzVYDcqovIrs6\nF1/XXILVYQUAhGpDcFfMNIyNGo200CFQKpSS1bR06L345kQ+PizYgRFhQyW9NhERERFRf6LTKHlO\n7A28J2n5COeaWJlDrNVhw9GykzhXdQH59YVwiA4AQIx/NMZGjcbYqNFIDEro8ZpWd4kPjEVW7EQc\nLT+JYxWncEfcFFnqICIiIiLydjqNCkaTVe4yvAZDrJsZvWR34qNlJ/Fe3ocAgKTgwRgbORpjo9Ix\nKCBa1rpu9K0hd+P09XPYUbgHE6LHQafSyl0SEREREZHX0WmUqNOb5C7DazDEupm3HLGTV1cAAPj5\npKcx2Et3AA7VhmBO4gzsuroX+4q/xL0p8+QuiYiIiIjI6/hplLDYHLA7HFAq5Jmk9CZ8B9zMaLZB\no1JArZLvrRVFEQX1VxCqDUFCYKxsdXTH3MQZCNIEYm/RQTSY9XKXQ0RERETkddrOijVzh2IAEofY\n48ePIysrC6tWrcKqVavwwgsvoLy8HKtWrcLKlSvx4x//GBaLRcqS3M5gssnehb1uqEKjtQlDQ1O8\n/vganUqLb6XcDYvDih2Fe+Quh4iIiIjI6/Cs2JtJ3i6cPHkyNm/ejM2bN+O5557Dq6++ipUrV+Kf\n//wnkpKSsG3bNqlLciuD2QZ/mY/Xya8vBACkhQ6RtY7umho7CYMCYnC0/CRKm8rlLoeIiIiIyKu0\nhVgjQywALxgnPn78OObMmQMAmDVrFo4ePSpzRb0niiKMZvk7sQWtIXZoPwmxSoUSS1MXQoSIjwp2\nyl0OEREREZFXaRsn5lmxLSQPsQUFBXjsscewYsUKHD58GEajERqNBgAQERGBqqoqqUtyG6vNAZtd\nlPV4nbb1sEHqQMT4R8lWR0+NjhiB4WFD8XXtJVysyZO7HCIiIiIir8Fx4ptJmraSk5PxxBNP4J57\n7kFxcTEefvhh2O3t/yFEUez2a0VFBXmixD5p2/Y6NFgnW33Xm6pQb25AVsJ4REcHy1JDb62ZtBz/\nvedFfHJ1F+4clgkFd17zyvucyN14n9NAwPucBgLe554TER4AANDq1HyfIXGIjYmJwcKFCwEAiYmJ\niIyMxIULF2AymaDT6XD9+nVER3fvHNOqqkZPltor5TXNAAClIF99x8tzAACD/QZ75XvUlQCEYvKg\n8ThecRo7LhzE1LhJcpckq6iooH7335Cop3if00DA+5wGAt7nnmW3towRX69qGjDvc1dhXdJW1/bt\n2/Hmm28CAKqqqlBTU4P7778fu3fvBgDs2bMH06dPl7Ikt/KGM2IL6lo3dQrrH+thb7VoyHyoFSp8\nUrgbZnv/3qmaiIiIiMgdOE58M0lD7OzZs3Hy5EmsXLkSa9euxW9+8xv85Cc/wUcffYSVK1eivr4e\nS5YskbIktzKaWkKsnGtiC+oL4a/yQ2xAjGw19EWYLhRzBt+FBose+699KXc5RERERESy48ZON5M0\nbQUGBuL111+/7fFNmzZJWYbHyN2JrTPVo9pUizGRo6AQ+u960nlJM3G47AT2XDuIaXFTEKLl3D8R\nERERDVzsxN6s/yYdL9QWYuU6J7ag/goAYGhoiizXdxedSod7h8yDxW7Bzit75C6HiIiIiEhWzhBr\nZogFGGLdyihzJza/9XzYtH5yPmxXpsVORox/NA6XnUB583W5yyEiIiIiko1Oy3HiGzHEupFB5jWx\nBfVXoFVqkBAYJ8v13UmpUGLp0IUQIeKjgk/lLoeIiIiISDYcJ74ZQ6wbtXVi5QixeksjrhsqMSQk\nGUqFUvLre0J6xEikhQ5BTs03+KY2X+5yiIiIiIhk4ecMsezEAgyxbiXnmti29bC+MErcRhAE3D/0\nWwCADws+hUN0yFwREREREZH0VEoFlAqBndhWDLFu1HbEjhxrYts3dfKdEAsAicEJmBQzHiVNZThZ\ncVbucoiIiIiIJCcIAnQaJUNsK4ZYNzKYbRAEQKuRfpy3oL4QaoUKScEJkl/b0xanzodKocL2ws9g\nsVvkLoeIiIiISHItIZbjxABDrFsZzTb4a1VQCIKk1222GlDWVIGU4CSoFPJsKuVJ4bowzB48HfXm\nBuwv/krucoiIiIiIJKfTqNiJbcUQ60YGs02WUeLL9VcgQuz358N25e6kmQhUB2BP0X7oLY1yl0NE\nREREJKm2cWJRFOUuRXYMsW5kMNlk2ZnYualTmG+th72Rn8oPC1PmwWy3YOeVvXKXQ0REREQkKZ1G\nCbtDhM3OzU4ZYt3E4RBhsthl6cTm1xdCKSiRHJwo+bWldGfcFET7R+Jw2XFUNFfKXQ4RERERkWR0\nmpacYeRIMUOsuxgt8hyvY7KZUNxYiqTgwdAoNZJeW2pKhRJLUu+FQ3Tgo8s75S6HiIiIiEgyOudZ\nsQyxDLFuItfxOpcbinx+PeyNMiJHYWhoCi5Uf428ustyl0NEREREJAlda84wmblDMUOsmxhabyap\n18QW1BcCANJ87HzYzgiCgPuHfgsA8J+CHXCIXBNARERERL6Pndh2DLFuYjTL04ktqL8CAQKGhCRJ\nel05JQUPxsSYcShuLMXx8tNyl0NERERE5HHtIZadWIZYNzGYpF8Ta7FbUKQvxuCgeOhUOsmu6w2W\npC6ERqHGx5d3wWA1yl0OEREREZFHtW3sxE4sQ6zbGGToxF7VX4NdtA+YUeIbhelCMT95DhqtTdh5\n5XO5yyEiIiIi8iiOE7djiHUTOdbE5te1rIcdKJs63WpO4l2I9IvAF6VHUNZUIXc5REREREQe4+zE\ncmMnhlh3ca6JlXCcuG097EANsWqFCg+kLYZDdGBr3scQRVHukoiIiIiIPEKnZSe2DUOsmzjXxErU\nibU6bLiiL0Jc4CD4q/0luaY3So8cifSIEcirv4wzleflLoeIiIiIyCM4TtyOIdZNjBKPE1/Tl8Dq\nsGHoAFwPe6tvpy2GSlDiPwU7YLZb5C6HiIiIiMjt2jd24jgxQ6ybGCQeJ247H3agjhLfKNo/EnMS\nZ6De3IA9V/fLXQ4RERERkdv5sRPrxBDrJlJ3YvMZYm8yP3k2QrUh2HvtC1QaquUuh4iIiIjIrThO\n3I4h1k0MJhs0KgVUSs+/pXaHHYUNVxHjH41gTZDHr9cfaJUa3D/0XthEOz7I/0TucoiIiIiI3Err\nDLEcJ2aIdROD2SbZKHFJUxnMdgu7sLcYHz0WaaFDkFNzETnVF+Uuh4iIiIjIbZQKBTQqBYzsxDLE\nuovRbJN8lDiNmzrdRBAELB+2BApBgW3522F18F+piIiIiMh36DRKjhODIdYtRFGEwSRdiOWmTp2L\nCxyEu+KnospYg/3XvpS7HCIiIiIit9FpVBwnBkOsW1htDtgdIvwkCLEO0YGC+quI1IUjTBfq8ev1\nR/em3I1AdQA+u7oPdaZ6ucshIiIiInILnZadWIAh1i3ajtfxl2BNbFlTBYw2I8+H7YK/2g/3pS6E\nxWHFhwWfyl0OEREREZFb6DQqmC12OERR7lJkxRDrBm3H60jRiS2ovwKAo8SuZMVOQFLwYJyuzEZe\n3WW5yyEiIiIi6rO2Y3bMA7wbyxDrBgaTdGfEtq2HTQtjJ7YrCkGB7wxbAgECtuZ9DLtjYP9B9zbn\nq3JxqbYA4gD/V0QiIiKinuBZsS2k2YnIx0nViRVFEQX1VxCqDUGELtyj1/IFScGDMTV2Io6Un8Sh\n0mOYOfgOuUsiAIdKj+Hfl/4DABgSkoSFKfMwIiwNgiDIXBkRERGRd9NpWvJGy+ZOWnmLkRE7sW4g\n1ZrY64YqNFqbMDQ0hT/wd9Pi1Hvgp9Jhx5XdaLQ0yV3OgJddlYv3Ln2IQHUAxkSOQmFDEV4793f8\n8fQGXKzJY2eWiIiIqAvsxLZgiHUDg0Sd2Hzn0TocJe6uIE0g7k25G0abCdsv75K7nAHtcv1VbMrd\nArVChbVj/wuPZTyCn0/6McZGjsYVfRFey2aYJSIiIuqKM8SaB/YxOxwndgOjRGtinethualTj9wV\nPxVHyk7gaPkp3BE/BcnBiXKX1CWL3YITFWcgVtmQ6jcUcYGD5C6pz8qbr+P185tgFx14LONRJAUP\nBgAMDorHDzJWo7ixFLuu7EV2dS5ey/47UoKTcG/KPIwI55gxERERUZv2ceKB3YlliHUDKTqxbeth\nA9UBiPGP9th1fJFSocTyYffhz2ffwPuXPsbPJj4OheB9QwhNlmZ8UXoEX5YcQZO12fn4IP9oZEZn\nYHx0BmIDYvpdqKsz1WP9uTdhsBnx8MjvYHTE8NuewzBLRERE5JpOy3FigCHWLRqaLQA8uya2xlSL\nenMDxkWN4Q/zvZAWlooJ0WNxujIbx8pPY1rcJLlLcqo21mJ/8Zc4UnYSVocV/io/LEieg7SYRHxZ\neBJf13yDXVf3YtfVvYjxj8b46DHIjM5AXMAgr78XDFYDNmT/A3XmetyXeg+mxE7o8vntYbYMu67u\nRXZVTmuYTcTClHkYGT7M679nIiIiIk9pXxPLcWLqg0aDBScvViIkUIOYMH+PXSe/rm2UmOthe2vp\n0HtxofprfHx5J8ZFpcNf7SdrPdf0Jdh77QucqTwPESLCdWGYPXg6psZOgk6lRVRUEEYEjITJZkZu\nzUWcqbyA3JpvsOvqPuy6ug8x/lHIjGoJtPGBsV4X7qx2K14//zbKmiswM+EOzEuc2e2vHRwUhx+M\nefimMLs++02GWSIiIhrQOE7cgiG2jz4/VQyz1Y777xoCtcpzI6oF9VcAAEO5HrbXwnShWJA8B9sL\nP8OnV/bggWH3SV6DKIq4WJuHz699gby6AgBAQmAc5iXOQGZ0BpQK5W1fo1NpMSFmHCbEjGsNtN/g\nbOV55NR8g8+K9uOzov2I9otEZnQGMqMzkOAFgdYhOvDW1//C5YYryIzOwLfTFvWqJoZZIiIionZ+\nrZ1YI0Ms9VaT0Yq9p0oQHKDBjHFxHr1WQX0h/FR+PrHJj5xmJ96Fo+Un8WXpUdwRN0Wy99PusON0\nZTb2XvsCpU3lAIARYWmYmzSjR2ektgTasZgQMxZmuwW5Nd/gTOV55FZfxO6i/dhdtB9RfhHONbQJ\ngXGShzxRFLE172Ocq8pBWugQrB75nT6vQW4LsyWtYfZca5hNDk7EguTZSAlJQoDKn4GWiIiIfNrN\n58QOXAyxfbD3VDFMFjvuuzMFGvXtHTR3qTPVo9pUizGRI71yQ6L+RK1QYVnaYvz1/Ca8n/cRfpz5\nQ48GH5PNjCPlJ7D/2iHUmeuhEBSYGDMOcxNnYHBQfJ9eW6vUYHxrWG0LtGcrzyOn+iL2FB3AnqID\nzkA7PT4L4bowN31XXdtdtB9flh5FfGAsfpixGmql2m2vnRAUh+/fEmZfP/8WAECj1CBCF4Zw56/Q\nmz4O1gTxzw8RERH1azwntgVDbC8ZTFZ8fqoEQf5qzBzXtzDiSvsoMdfDukN65EikR4xETs1FnKnM\nxoSYcW6/ht7SiIPFh3Go9CgMNiM0CjVmJtyB2YOnI8Iv3O3XuzHQWuwW5NZcwtnK87hQ0xJoDxQf\nwpzEGZiXOBM6ldbt129zpOwkPincjTBtKNaO/S/4qTyz7vjGMHus4hSqjbWoNdWh1lSP8ubrHX6N\nSlAi7JZgG64LcwbfUG1Ih+PcRERERN5Cp+WaWIAhttf2ni6B0WzDAzNTodV49gff/Hpu6uRu305b\nhG9q8/Cfgk+RHjkKWqXGLa973VCFfde+xPGK07A5bAhUB+BbKXdjesJUBKoD3HINVzRKDTKjxyAz\negwsditOV2bjk8uf4bOr+3C07CTuS70HkwZlur0rmVN9Ef+69AECVP54Ytz3EKoNcevrdyQhKA7L\nghbf9JjRZkStqR61pjrUmOpawq2xzvnYpda1yLcSIGBUxHB8f8zDUCv4VyMRERF5H+5O3ELyn9Re\nfvllnD59GjabDT/84Q+xf/9+5ObmIjQ0FACwZs0azJw5U+qyesRotuHzk8UI9FNj1njPdmGBlk6s\nVqlBQqBn190OJNH+kZibOAOfFe3H7qv7sTh1QZfPd4gONFmbUW9uQINZj3pzA+pNDahv+9jc8rHJ\nbgIARPpFYG7iXZgyaCI0bhyn7SmNUo2psRORGTUGe68dxN5rX+Cdi+/hi9IjWJa2GENCktxynSsN\n1/D3nHehFJR4bOyjGBQg31nGfio/xAf6IT4wtsPft9itqGvt2taYap3htqSpDLk13+Djgp1YNmxx\nh19LREREJCeNSgFBAExmdmIlc+zYMeTn5+O9995DXV0dli5diqysLPy///f/MGvWLClL6ZN9p0vQ\nbLLh/ruwrWCxAAAgAElEQVSGOBdXe4re0ojrhkqMDB/GUUc3uzt5No5XnMG+a19gVMRwAHAG0oZb\nwmmDWQ+72PlfFgEqf+d46pTYCRgXle5V6y91Ki2+NWQ+psZOxseXd+J0ZTb+eHo9JsaMw5LUhQjT\nhfb6ta83V+Kv5/8Bu2jHD8Y87LZg7CkapRoxAdGIuSVom+0WvHTyVRwo+QrDw4diTOQomSokIiIi\n6pggCNBpVOzESnmxSZMmISMjAwAQHBwMo9EIu71//SuC0WzD7hPXEKBTYc6EBI9fj+thPUer1GDp\n0Hvxj9wt+P/O/LXD5ygEBYI1QRgcFI9QbQhCtcEI1YYgRBuMMG0IQlof07hpHNnTIvzC8F/pD2JG\n/R3Ylv8xTl0/h+yqXMxNnIF5STN7PFZdb27Aa9lvotlqwIMjlvXr4KdVarAm/UG8fOov2Hzxffxy\n8k8kGYkmIiIi6gmdRsk1sVJeTKlUwt/fHwCwbds23HXXXVAqlXj33XexadMmRERE4LnnnkN4uOuN\nb6Kigjxdboc+2J+PZpMNDy4YgcQEz+/2WnqtBAAwKTldtu/Zl82PvAP1Yg2qm+sQ7h+KcL8bfvmH\nIlQbDIVCvo6qp/6bR0WNweSho/Hl1eP41/mPsevqXhy/fgoPZizFHUkTu9VFNliMePnA26g11eE7\n6Ytw3+g5HqlVSlFRQVhtW4Y3z/wbW/Lex/Mzn5b1v/9Awb/baCDgfU4DAe9zaQT6q1HfaBnQ77cs\nu5fs3bsX27Ztwz/+8Q/k5OQgNDQUI0eOxN/+9je89tpreP75512+RlVVowSV3sxsseODA/nw06ow\ndUSUJDVcqLgEtUKFYEe4LN/zQDBn0OzbH3QA9iagpqlZ+oJaRUUFefy/+ejAdDw7OQ17ig5gX/GX\n+MvxTfjk4j4sS1uMlJDETr/O6rBhw7k3UVRfgunxUzE96k6fuT8zQzIxNioH2VU5ePfUx7gnZa7c\nJfk0Ke5zIrnxPqeBgPe5dFQKBYxmq8+/312FdMlbDIcOHcLrr7+OjRs3IigoCFOnTsXIkSMBALNn\nz0ZeXp7UJXXbgbOlaDRYMW9iAvx1nt+sp9lqQFlTBZKDE7lbKnmMTqXF4tQFeH7Kz5AZnYGr+mv4\nw+nX8Fbuv1Fnqr/t+Q7RgXe+/jfy6i9jXFQ6lg+7z6Nn7UpNEAQ8OGIZwrSh+PTK586RfiIiIiJv\noNMoYbOLsNkdcpciG0lDbGNjI15++WW88cYbzt2In3zySRQXFwMAjh8/jrS0NClL6jaz1Y7PTlyD\nTqPEvEmDJbnm5forECHyaB2SRIRfOL6X/hCeznwMgwPjcPL6Gfz22CvYeeVzWOwWAIAoivgg/xOc\nqTyP1JAUPDJqhVdtYOUuAWp/PDJ6BQDgrdx/odlqkLkiIiIiohbtx+wM3HWxkrb3du7cibq6Ojz9\n9NPOx+6//348/fTT8PPzg7+/P1588UUpS+q2L8+VQd9swbemJSFAgi4swE2dSB5pYUOwbtJTOFZ+\nGtsLd+HTK5/jSNlJLEm9B7XmehwsOYzYgBg8lrEaahmPD/K0oaEpuDdlHnZc2YMtF7fi+2Me9qmO\nMxEREfVPbaejmMw2BPr57s9iXZE0xH7nO9/Bd77zndseX7p0qZRl9JjVZsfO40XQqpW4e1Ln6wTd\nLb++EEpB2eXaRCJPUAgKTIubhPHRY7C76AD2X/sSm77+FwAgVBuCx8eugb/aX+YqPW9+8mxcqitA\ndnUuDpUexV0J0+QuiYiIiAY4nZadWN+bA/SAL7PL0dBkwewJ8ZL9a4fJZkJxYymSghP6zfEt5Ht0\nKh3uS70Hz2X9DJlRYxCpC8fjY9f06VzZ/kQhKPDI6BUIUPvjg4IdKGksk7skIiIiGuA4TswQ65LV\n5sDOY0XQqBWYP1m6jujlhiKIEDlKTF4h0i8C3xuzCv8z7eeICxwkdzmSCtWGYNXI5bA5bPhH7j9h\nbl0fTERERCQH5zixxSZzJfJhiHXhqwvlqGs0Y3ZmAoL9peuIFtQXAuB6WCJvMCZyFGYl3Inrhkps\ny/tY7nJ6pdJQhSarfEdGERERkXuwE8sQ2yWb3YGdR69CrVJg/hRp16UW1F+BAAFDQpIkvS4Rdey+\noQsxODAOR8pP4tT1c3KX022iKGJ/8SG8cPyP+P2J/0OtqU7ukoiIiKgP2kKskZ1Y6sjhC+Wo0Zsx\nc1w8QgKk68Ja7BYU6YsxOCgefiqdZNclos6pFSo8mv4gNEoN/vXNB6g21shdkksWuxXvXHwPH+R/\nAo1Cjf+/vXsPj6LO8z3+ruprLuQGCRBFQIiEAYQBYUTU9TaOunrUM+Oio8fB4zgX1plz2GcGeRj3\nAR8OqIzPswycOerqwzgO44oTldVRwUUBHY0gogjILUEC4ZKQCyEh6U53V50/On1JCAxCku5OPi+J\nVfWr6u5fd32rur79+1VVvf84y754jsbWpkRXTURERM5RWrQ7sVpipYNgyOKt0gqcDpObergVdv+J\nA4TsECNzhvfo64rImQ1Mz+fuS+7EF/KzfMdLBK3k/QW0zlfPv235f2w6uoVhWRfxr5f/iu9edA3V\nzTX83y+epznQkugqioiIyDmIXp3Yn7zHId1NSexplO44Sk2Dj3+YUEhuP0+Pvvbe+vD5sEU6H1Yk\n6Xxn8CSmDJpIxYmDvLlvTaKr06m99ft48tOlHGg8xNTBk/nfE39Gjieb20fczJWF36Gy6TDPfPkH\nWnWRKhERkZTjVUusktjOhCyLtz6uwOkwuLmHW2Ehdj7sCLXEiiSl6ZfcQX5af9Ye2MBXtbsTXZ0o\n27bZUPkxS7/4d5qDLUy/5A7uLf4BLjP8ZWcYBtNH3cmkgvGUN+znuW1/SurWZBERETmVLuykJLZT\nG7+qovp4C1ddWkheVs+ekxqwgnx9ooLCzEFkuNJ79LVF5Ox4nV7+55h7cRgOXvxqJQ3+xkRXiYAV\n5M+7SnhlzyrSnWn8csJPuPrCKzAMo91ypmFy/7emM6Z/MV/V7eaPX72MZVsJqrWIiIh8U7Ektu/+\nEK0ktgPLsnnz4wocpsEtl/f8lYEPnKgkYAV1PqxIkrso60LuGHkLjYEmXkxwInjc38CSLc9QeuRT\nLup3AXMm/y+Kck9/OoLTdPLjsfcxIns4W6q/5OXdr2Hbdg/WWERERM6VuhMriT3Fpp1VVNU1c+Wl\ng+mf3fNXBt6r+8OKpIxrL7ySsf2L2VW/l7UHNiSkDvsa9vPkp0vZf+IAUwZNZNbEmeR6c/7u49wO\nNz8fP4Mh/S7go8ObWFX+thJZERGRFKDuxEpi2wm3wu7HYRr8YwJaYQHKokmsWmJFkp1hGNw3+p/I\ndvfjzX1r+Lqhokdf/6NDG1my5VmaAif5QdF/4/7R03E7XGf9+DRnGv88/kEGpuez9sAG3q1Y1421\nFRERka7gdJg4Haa6E0vY5t3VHKltZurYQQzISevR125sbWJV2dvsqS9nYHo+We5+Pfr6InJu+rkz\nmTHmHmzb5g87XuqRW9cErSD/sfs1Xtr9Kl6nh38e/yDXDrnylPNfz0Y/dya/mPAQuZ4c3ti3mg8q\nS7uhxiIiItKVvG6HWmJTzZpPKmjp4vsiWbbNmx/txzQMbp3ac62wx/0NlOx5g3/9+HH+68B6Mlzp\n/PeRt/bY64vI+bskdyTfG3Ydtb56/mP3q93aLbfB38jvPv93/nboEy7IHMzsy35JcV7ReT1nrjeH\nX3z7Ifq5Mnllzyo2H/28i2orIiIi3aGvJ7HORFfgXDz9wSrcjUP4h7HDuX7ShV1y7uqW3cc4VHOS\nK8YOoiC3+68KXNtSx7sH1vPJ4U8J2iFyPTncOPQapg6ejOsbdAcUkeRwy7Ab2FNfzpbqLynOK2Ja\n4Xe6/DX2nzjAc9v+xHF/A5MKxnPv6LvwONxd8twD0/N5eMKPWfL5M/xx50o8Tg/jBnyrS55bRERE\nulZmmosDVU2s3niAGycPwTS/eW+sVOaYP3/+/ERX4pt6rfKPMGA/5bWH+a+Pazh0JEheP8853w7H\nsm2efeMrGlta+fkdY8lM674ksqr5GK/vfYs/7y6h4sRB8tLyuHPkrdw7+gcMzx6Kw3R022tLasnI\n8NDc3JroashZMg2TUbkj+eToZ2yv2YnH4aEp0EQgFMBpOnCaznPq7htRemQzz2//Ey1BH3eMuIXv\nF92G0+za3yGzPP0YkT2cT6s+5/PqLxmRPYz+aXld+hodKc6lL1CcS1+gOO9Z+TlpfLmvls/31rBt\nXy0jCrPJyuiaH7aTRUaG57TzDDsFL0f59p73eWvXOo611ABgNWUTrBrKUE8R35synImXDMBhnn1P\n6c/3HGPZa9u4fMxAfnLbmG6p8+Gmo6ypeJ/PqrZiYzMovYDvDbuOSQXjlbhKp/Lz+3HsWOLvPyrf\nzBfV23h++wps2u9aPQ43ud5c8jw55HpzyPPmkOtpG3pzyPFkd5qUhqwQr5X9lfWVH5HmTOOBMT9k\nTP9R3foevqrdzTNfvoDLdPLLb/+EoVlDuu21FOfSFyjOpS9QnPe8xuZWXn5vL6U7qsIXpp06lH+c\nOgyXMyXPGD1Ffv7prxGUkkksQFV1Azvr9rLh4EfsqNsFgN3qIXjsQrKai7hhwkiuHl9IuvfMLRW2\nbfPYC59ysKqJBT/+DoUDMrq0ngdOVLK64n22HtsOwIWZhdw07HrG54/BNHpHgEn30JdB6jpysopD\nTUeo9x2nznecen99eOg7TnOw8ws/GRhkuTPJ9eaGk9y2ZHfrse3sPb6PwRkD+cm4H1GQPqBH3sOW\n6i9Zvv3PpLvS+JeJP2dQxsBueR3FufQFinPpCxTnifNleQ1/XL2b+kY/FwzIYMYtxYwozE50tc5b\nr0xi4zeS6uYaPjxUykeHNuG3/Ni2QahuEGbtcK4cOZrvXnYRBae52vAXZTUsLfmSKaML+NntY7us\nfvsaKli9/z121IYT7GFZF3HTsOsY23/0eXUplL5DXwa9ky/oo97f0JbU1ocTXf/xuIT3OJZttXvM\n+Pyx3D/6n/A6e/be1R8f3sSfd5WQ48nmXyb+vFu6FivOpS9QnEtfoDhPrBZ/kJIN5azbcggD+O7k\nIdx51cV43Knb47PXJ7ERvqCfT6u28P6Bj6huqQbAOplFsGoo4/LGcdPkYRRdmB1NIm3b5v+8uJmv\njzSy4MEpXJCfeV51sm2bvcf38c7+99hTXwaE7/d687AbGJU7UsmrfCP6MuibLNviRGsj9b7j1Psb\ncBgOxg0YnbCeG2sPbOD1srfIT+vPrIkzyfZ07e2/FOfSFyjOpS9QnCeH3QfqeeGdXVTVtzAg28uM\nm4v51rDuvb5Fd+kzSWyEbdvsqS9n3cG/sb12JzY2dsBF8NgQBtujuXniKC4rLmBnRT3/9spWLhuV\nz8w7x51zXUJWiF31Zaze/x77GvYDMDrvEr439DqKci8+5+eVvk1fBpIs3ixfzeqK9ynMGMSsiT8j\n3XV+V3BvDQU40XqC4/4TDOyfTUYwR6dXSK+m/bn0BYrz5NEaCPGfH33Nmo0HsWybqy4dzPTrRpLu\nTa07oPS5JDZebUsdHxwq5W+Vm/BZLdi2gVVfQFrjSNz+fI4d9zH/gclcNDD8IbWGApwMnKQp0MzJ\nwMm2v+boX1PcdGTcF/JFX2/cgG9x07DrGJZ1Ube8b+k79GUgycK2bV7Z8598cOhjhmcN5RfffqjT\nW/uErBAnWhs57j9BQ+sJGvyxv+P+hmhZx/OC05xpjModwajcIorzRpKfNkA9V6RX0f5c+gLFefLZ\nf/QEf3h7Fwerm8jOdPM/bhzFxEvyE12ts9ank9iI1lCAzVWfs7bib1S1HAXAas4k05VJXq4ZTUgD\nVuCsns9lOslwZZDhSifDlcEAby5XXziNIf0Kv/F7EemMvgwkmVi2xYtfvcKnVVsoyrmYopyLaWhr\nTY0kqk2Bk6dclTleujONbE8W2e6s8NCTRdDRytbDX1Hrq48ul+vJYVTeSIpzixiVN5Isd9d2YRbp\nadqfS1+gOE9OwZDF6o0HeOOjrwmGbC4rLuDe715CdgrcjkdJbBzbtilv2M97FR+yrXYHNjYeh5vM\nuIQ0fhgrj41nujJwd9IKIdKV9GUgySZkhXhu+4tsq9nZrtztcJPTITnNiY5nk+PJIsudhdtxajem\nSJzXtNSyq24vu+rL2FNXxslgc3SZwoxBFOcVMSp3JCNzLsbrPP1940SSkfbn0hcozpPbkdqT/OHt\nXZQdaiDD6+Tu64u4YuygpO75pCT2NFpDrRiGiauTezOKJJq+DCQZBa0gu+vLcBrOaMLqdXjO+Uuw\nszi3bIvKpsPsritjV91eyhu+JmAFATANk+FZQynOG8mo3CKGZQ3RvbYl6Z3v/ty2bfyhVlqtVkxM\nXA4XLtOpc8klqei4JflZts26LYcoWV+OPxBizPBc7rnxYjIzwvmQx+FJqv2KkliRFKQvA+kLzibO\nA6EA+xoq2FW/l931ZRw4URnttuxxuCnKGcGovJFku7Nwmg4chgOn6WwbOnCYDpyGs23Y+fSZvrRt\n28ayLUJ2iFBkaFlYdihWZoWH0TIrvByAaRgYGBiGGR03DbOtrMN429DAbJtnROedqX5n0rGLt2Vb\nBK0gAStE0AqG/+zwMNRWFrDj5llBglYoukzsMSEchgO36YomVS7TFf5zxI2bTtyO8LjTdOE2nW3L\nxx7jMB3Rz9nCxrat8LhtY2HF5rWV2VjtpsOPs6KfRfizNNt9vma7cTP6OZtG/Gfffr5t2+GLQ9oW\nNnb4tdvKIvUI1y2yXKy+kWkbm8wsN0dr6vGF/Pjb/nxBP/5Qa4fpcJkv5MffNu0L+WkNBTrtqu8w\nHLhMJ87IZ9/hc48OHS6cphO3GT+MX97Zbr6r3Tp1Rtdd9LkMx1n9cBXbdsLbgxU/jG4zsTLLtjAN\nE9MwcRiO6NBhxk+bmIajbWgmVStSZH3Hj4eHtIsHOxrnkVhqHzPxsRWLtUj8WdHXaBfD0X1HbNh5\neefLm4aJGRf755LInMtxy+m2++5yargYZ5jqWBK/Lqx26y62X+qwv4hfJm6ddrZeDDjNujPbl8fN\nC1khfCE/vqAPX9BPSyg89IV80TJfqMO8oI/mgI8TvmZCRgDDaP95e0wPbocHj+mJjrsj46YHt+nG\nZXpwG+Fxt+HBZXhwGW7cpgdHW+OgZdsYkei3bSwjPE4k1o248eh/kc/ZwjDgzqlTTr8ulcSKJCcl\nsdIXnEucNwea2XN8H7vr9rKrfi/VzTXnXY/IwbLTdGBitiWnsYNs6V4GxhnPp+5r3A43XocHr8OD\nx+HG4/TgcXhwO9zYtkXACob/QgGCVoBWK0jQCrSVh4fBtt4L3SU+SQbj1CS1h7adaKIbn/iajnYx\nFTvUjSWU4X+xmOtYFvl/+KGnJqTYNlaHhLW3iSS2kR/gwtOxJDf+RyHTMHA6HASCQaxo4hb7ISqS\n3EUS1vjEXXqOgYHX6Q3vX5xeAn6DmrogoaAJhgWOIIYjGB4624ZG4tbRK9OfPu089aMVEZGUku5K\nZ0L+WCbkjwWgzldP2fGvaQn6CFlBQrZF0AoRsoNtw9CZp60QQTsyDGLZVttBcbgFKHaQHJmOHTB3\nWmbGtRJhRFsY7LaD3na/yJ+mLP6X/khLxZlanIxT2g86tC7ETZqYOM1wa3X0z3DGlTnapp2x6egy\nkVY/Bw7DScgOtSVNAQKhcALVagUIhALRZCoQnQ5Gl21tWzaSgIWsYFsLUKw1KNziYHbaQhQ5mI62\nrkYOsgm3HlkdWkijB9SdtHRZ7VpNIgfbnbeWnK7V3OyklSTW2mvQLyMNq9XAE0lMneHkNJykhqcj\nCavb4e6S7nxW23YQbFsnQStIayjQ1gofWxeRZDg++Y0sf8p6jJZF1l94WQCv4Y3GfSyZjN822ieZ\n7Vtbw+OR1u/4Xg+RVttYYhyZ1zZuWR2S5/B827agrXULM7aFGBiE/xnttxzjlJLoNheLAzAwwYg8\nc6yctvUfflx0bjQGIq9pnqY1ND5e4nsSxPcSMOOfj1jrbnyrrh2/32hXbnda3rGV8NQeD+23n+i8\nuIQ0vE+1CNgG2OH6OQwHZluX9063647bb4f5p+7Tzt+pCXP76VOb9ex2Yx33A2aHdRd5T4YRv77i\n1lvcev176+Vs16OJQZrT25aYevE6w8lpWluSmhaXsHqdXtym65TvkoYmP2s+PUizLxCL80hcW2AT\nwjIChIzW8JBWQkYrIQLRYZDY0CL8A5pBLB6IxK0di9/YFhceRsbjtrozrk+1xIokKbXESl+gOJe+\nQHEufYHiXLramc6JTZ4zd0VERERERET+DiWxIiIiIiIikjKUxIqIiIiIiEjKUBIrIiIiIiIiKUNJ\nrIiIiIiIiKQMJbEiIiIiIiKSMpTEioiIiIiISMpQEisiIiIiIiIpQ0msiIiIiIiIpAwlsSIiIiIi\nIpIylMSKiIiIiIhIylASKyIiIiIiIilDSayIiIiIiIikDCWxIiIiIiIikjKcia5AxKJFi9i6dSuG\nYTB37lwuvfTSRFdJREREREREkkxSJLGbNm2ioqKClStXUl5ezty5c1m5cmWiqyUiIiIiIiJJJim6\nE5eWlnLDDTcAMGLECBoaGmhqakpwrURERERERCTZJEVLbE1NDWPGjIlO5+XlcezYMTIzM0/7mPz8\nfj1RNZGEUpxLX6A4l75AcS59geJcekpStMR2ZNt2oqsgIiIiIiIiSSgpktiCggJqamqi09XV1eTn\n5yewRiIiIiIiIpKMkiKJnTZtGmvWrAFgx44dFBQUnLErsYiIiIiIiPRNSXFO7MSJExkzZgx33303\nhmEwb968RFdJREREREREkpBh6wRUERERERERSRFJ0Z1YRERERERE5GwoiRUREREREZGUkRTnxH4T\nixYtYuvWrRiGwdy5c7n00ksTXSWRLrFnzx5mzpzJjBkzuO+++zhy5AizZ88mFAqRn5/Pb3/7W9xu\nd6KrKXJeFi9ezGeffUYwGOSnP/0p48aNU5xLr9LS0sKcOXOora3F7/czc+ZMiouLFefSK/l8Pm69\n9VZmzpzJ1KlTFefSY1KqJXbTpk1UVFSwcuVKFi5cyMKFCxNdJZEu0dzczIIFC5g6dWq0bOnSpfzw\nhz/kpZdeYujQoZSUlCSwhiLn75NPPmHv3r2sXLmS559/nkWLFinOpddZt24dY8eOZcWKFSxZsoQn\nnnhCcS691tNPP012djag4xbpWSmVxJaWlnLDDTcAMGLECBoaGmhqakpwrUTOn9vt5rnnnqOgoCBa\ntnHjRq6//noArr32WkpLSxNVPZEuMXnyZH73u98BkJWVRUtLi+Jcep1bbrmFhx56CIAjR44wcOBA\nxbn0SuXl5ZSVlXHNNdcAOm6RnpVSSWxNTQ25ubnR6by8PI4dO5bAGol0DafTidfrbVfW0tIS7YbT\nv39/xbqkPIfDQXp6OgAlJSVcffXVinPpte6++25+9atfMXfuXMW59EpPPvkkc+bMiU4rzqUnpdw5\nsfF0dyDpKxTr0pusXbuWkpISli9fzo033hgtV5xLb/Lyyy+zc+dOfv3rX7eLbcW59AarVq1iwoQJ\nDBkypNP5inPpbimVxBYUFFBTUxOdrq6uJj8/P4E1Euk+6enp+Hw+vF4vVVVV7boai6SqDz/8kGee\neYbnn3+efv36Kc6l19m+fTv9+/dn8ODBjB49mlAoREZGhuJcepX169dz8OBB1q9fz9GjR3G73dqf\nS49Kqe7E06ZNY82aNQDs2LGDgoICMjMzE1wrke5xxRVXROP93Xff5aqrrkpwjUTOT2NjI4sXL+bZ\nZ58lJycHUJxL77N582aWL18OhE+Dam5uVpxLr7NkyRJeffVVXnnlFe666y5mzpypOJceZdgp1t7/\n1FNPsXnzZgzDYN68eRQXFye6SiLnbfv27Tz55JMcOnQIp9PJwIEDeeqpp5gzZw5+v5/CwkIef/xx\nXC5Xoqsqcs5WrlzJsmXLGD58eLTsiSee4NFHH1WcS6/h8/n4zW9+w5EjR/D5fDz88MOMHTuWRx55\nRHEuvdKyZcu44IILuPLKKxXn0mNSLokVERERERGRviuluhOLiIiIiIhI36YkVkRERERERFKGklgR\nERERERFJGUpiRUREREREJGUoiRUREREREZGUoSRWRESkG+zcuZMFCxZQVlbGjh07uuQ5q6qqKC0t\nBeC1117jL3/5S5c8r4iISCrRLXZERES60dNPP82AAQO46667zvu53njjDcrLy5k1a1YX1ExERCQ1\nORNdARERkd5o48aNzJgxg7y8PDIzM/F6vVx99dXMmzePuro6mpqaeOCBB7jttttYtmwZlZWVHD58\nmEceeQSfz8dTTz2F2+3G5/Mxb948srKyWLJkCbZtk5OTQ1NTE8FgkFmzZrF+/Xp+//vf4/V6SUtL\nY8GCBQwcOJDrrruO+++/nw8++IDKykoee+wxpk6dmuiPRkRE5LwoiRUREekmEyZMYOjQoUyaNInb\nbruNxx57jKuuuorvf//7NDc3c/vttzNt2jQAKisrWbFiBYZhsHbtWubPn09xcTF//etfefbZZ1m6\ndCl33nknwWCQBx54gGXLlgHQ0tLCo48+SklJCYMGDWLFihUsWbKExx9/HACPx8Py5ct5/fXXefHF\nF5XEiohIylMSKyIi0kM2btzItm3bWLVqFQBOp5PKykoAxo8fj2EYAAwYMIDFixfj9/tpbGwkOzv7\ntM+5f/9++vfvz6BBgwCYMmUKL7/8cnT+lClTACgsLKShoaFb3peIiEhPUhIrIiLSQ9xuN/PmzWPc\nuHHtyjds2IDL5YpOz549O9r1d926dSxfvvy0zxlJfCNs225X5nQ6280TERFJdbo6sYiISDcyDINA\nIADApEmTeOeddwDw+XzMnz+fYDB4ymNqamooKioiFAqxevVqWltbo8/Vcflhw4ZRW1vL4cOHASgt\nLTSs/wEAAADYSURBVGX8+PHd+ZZEREQSSi2xIiIi3ejyyy9n8eLF2LbNww8/zKOPPso999xDa2sr\n06dPb9dSGvHQQw/xox/9iMLCQh588EFmz57NCy+8wGWXXcasWbNwuVw4HA4AvF4vCxcuZNasWbjd\nbtLT01m4cGFPv00REZEeo1vsiIiIiIiISMpQd2IRERERERFJGUpiRUREREREJGUoiRUREREREZGU\noSRWREREREREUoaSWBEREREREUkZSmJFREREREQkZSiJFRERERERkZShJFZERERERERSxv8H1uXh\ng0JNipIAAAAASUVORK5CYII=\n",
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {
+ "tags": []
+ }
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/dopamine/colab/load_statistics.ipynb b/dopamine/colab/load_statistics.ipynb
index abd0b715..9d91858a 100644
--- a/dopamine/colab/load_statistics.ipynb
+++ b/dopamine/colab/load_statistics.ipynb
@@ -79,7 +79,7 @@
"import numpy as np\n",
"import os\n",
"from dopamine.agents.dqn import dqn_agent\n",
- "from dopamine.atari import run_experiment\n",
+ "from dopamine.discrete_domains import run_experiment\n",
"from dopamine.colab import utils as colab_utils\n",
"from absl import flags\n",
"\n",
@@ -310,4 +310,4 @@
]
}
]
-}
\ No newline at end of file
+}
diff --git a/dopamine/colab/utils.py b/dopamine/colab/utils.py
index 7a018130..86176cd8 100644
--- a/dopamine/colab/utils.py
+++ b/dopamine/colab/utils.py
@@ -72,6 +72,9 @@ def load_baselines(base_dir, verbose=False):
if sys.version_info.major >= 3:
# pylint: disable=unexpected-keyword-arg
single_agent_data = pickle.load(f, encoding='latin1')
+ cols = single_agent_data.columns
+ single_agent_data[cols] = single_agent_data[cols].apply(
+ pd.to_numeric, errors='ignore')
# pylint: enable=unexpected-keyword-arg
else:
single_agent_data = pickle.load(f)
@@ -167,7 +170,7 @@ def summarize_data(data, summary_keys):
Example:
data = load_statistics(...)
- get_iteration_summmary(data, ['train_episode_returns',
+ summarize_data(data, ['train_episode_returns',
'eval_episode_returns'])
Returns:
@@ -277,4 +280,8 @@ def read_experiment(log_path,
row_index += 1
# Shed any unused rows.
- return data_frame.drop(np.arange(row_index, expected_num_rows))
+ df = data_frame.drop(np.arange(row_index, expected_num_rows))
+
+ cols = df.columns
+ df[cols] = df[cols].apply(pd.to_numeric, errors='ignore')
+ return df
diff --git a/dopamine/common/__init__.py b/dopamine/common/__init__.py
index f9bcb7c5..77c9b2a6 100644
--- a/dopamine/common/__init__.py
+++ b/dopamine/common/__init__.py
@@ -12,3 +12,14 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+CHECKPOINT_DURATION = 4
+
+
+def set_checkpoint_duration(n):
+ global CHECKPOINT_DURATION
+ CHECKPOINT_DURATION = n
+
+
+def get_checkpoint_duration():
+ return CHECKPOINT_DURATION
diff --git a/dopamine/discrete_domains/__init__.py b/dopamine/discrete_domains/__init__.py
new file mode 100644
index 00000000..f9402b68
--- /dev/null
+++ b/dopamine/discrete_domains/__init__.py
@@ -0,0 +1,15 @@
+# coding=utf-8
+"""Copyright 2018 The Dopamine Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
diff --git a/dopamine/atari/preprocessing.py b/dopamine/discrete_domains/atari_lib.py
similarity index 53%
rename from dopamine/atari/preprocessing.py
rename to dopamine/discrete_domains/atari_lib.py
index 861c544a..487eac62 100644
--- a/dopamine/atari/preprocessing.py
+++ b/dopamine/discrete_domains/atari_lib.py
@@ -12,9 +12,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""A class implementing minimal Atari 2600 preprocessing.
+"""Atari-specific utilities including Atari-specific network architectures.
-This includes:
+This includes a class implementing minimal Atari 2600 preprocessing, which
+is in charge of:
. Emitting a terminal signal when losing a life (optional).
. Frame skipping and color pooling.
. Resizing the image before it is provided to the agent.
@@ -24,11 +25,183 @@
from __future__ import division
from __future__ import print_function
+import math
+
+
+import atari_py
+import gym
from gym.spaces.box import Box
import numpy as np
+import tensorflow as tf
+
import gin.tf
import cv2
+slim = tf.contrib.slim
+
+
+NATURE_DQN_OBSERVATION_SHAPE = (84, 84) # Size of downscaled Atari 2600 frame.
+NATURE_DQN_DTYPE = tf.uint8 # DType of Atari 2600 observations.
+NATURE_DQN_STACK_SIZE = 4 # Number of frames in the state stack.
+
+
+
+
+@gin.configurable
+def create_atari_environment(game_name=None, sticky_actions=True):
+ """Wraps an Atari 2600 Gym environment with some basic preprocessing.
+
+ This preprocessing matches the guidelines proposed in Machado et al. (2017),
+ "Revisiting the Arcade Learning Environment: Evaluation Protocols and Open
+ Problems for General Agents".
+
+ The created environment is the Gym wrapper around the Arcade Learning
+ Environment.
+
+ The main choice available to the user is whether to use sticky actions or not.
+ Sticky actions, as prescribed by Machado et al., cause actions to persist
+ with some probability (0.25) when a new command is sent to the ALE. This
+ can be viewed as introducing a mild form of stochasticity in the environment.
+ We use them by default.
+
+ Args:
+ game_name: str, the name of the Atari 2600 domain.
+ sticky_actions: bool, whether to use sticky_actions as per Machado et al.
+
+ Returns:
+ An Atari 2600 environment with some standard preprocessing.
+ """
+ assert game_name is not None
+ game_version = 'v0' if sticky_actions else 'v4'
+ full_game_name = '{}NoFrameskip-{}'.format(game_name, game_version)
+ env = gym.make(full_game_name)
+ # Strip out the TimeLimit wrapper from Gym, which caps us at 100k frames. We
+ # handle this time limit internally instead, which lets us cap at 108k frames
+ # (30 minutes). The TimeLimit wrapper also plays poorly with saving and
+ # restoring states.
+ env = env.env
+ env = AtariPreprocessing(env)
+ return env
+
+
+def nature_dqn_network(num_actions, network_type, state):
+ """The convolutional network used to compute the agent's Q-values.
+
+ Args:
+ num_actions: int, number of actions.
+ network_type: namedtuple, collection of expected values to return.
+ state: `tf.Tensor`, contains the agent's current state.
+
+ Returns:
+ net: _network_type object containing the tensors output by the network.
+ """
+ net = tf.cast(state, tf.float32)
+ net = tf.div(net, 255.)
+ net = slim.conv2d(net, 32, [8, 8], stride=4)
+ net = slim.conv2d(net, 64, [4, 4], stride=2)
+ net = slim.conv2d(net, 64, [3, 3], stride=1)
+ net = slim.flatten(net)
+ net = slim.fully_connected(net, 512)
+ q_values = slim.fully_connected(net, num_actions, activation_fn=None)
+ return network_type(q_values)
+
+
+def rainbow_network(num_actions, num_atoms, support, network_type, state):
+ """The convolutional network used to compute agent's Q-value distributions.
+
+ Args:
+ num_actions: int, number of actions.
+ num_atoms: int, the number of buckets of the value function distribution.
+ support: tf.linspace, the support of the Q-value distribution.
+ network_type: namedtuple, collection of expected values to return.
+ state: `tf.Tensor`, contains the agent's current state.
+
+ Returns:
+ net: _network_type object containing the tensors output by the network.
+ """
+ weights_initializer = slim.variance_scaling_initializer(
+ factor=1.0 / np.sqrt(3.0), mode='FAN_IN', uniform=True)
+
+ net = tf.cast(state, tf.float32)
+ net = tf.div(net, 255.)
+ net = slim.conv2d(
+ net, 32, [8, 8], stride=4, weights_initializer=weights_initializer)
+ net = slim.conv2d(
+ net, 64, [4, 4], stride=2, weights_initializer=weights_initializer)
+ net = slim.conv2d(
+ net, 64, [3, 3], stride=1, weights_initializer=weights_initializer)
+ net = slim.flatten(net)
+ net = slim.fully_connected(
+ net, 512, weights_initializer=weights_initializer)
+ net = slim.fully_connected(
+ net,
+ num_actions * num_atoms,
+ activation_fn=None,
+ weights_initializer=weights_initializer)
+
+ logits = tf.reshape(net, [-1, num_actions, num_atoms])
+ probabilities = tf.contrib.layers.softmax(logits)
+ q_values = tf.reduce_sum(support * probabilities, axis=2)
+ return network_type(q_values, logits, probabilities)
+
+
+def implicit_quantile_network(num_actions, quantile_embedding_dim,
+ network_type, state, num_quantiles):
+ """The Implicit Quantile ConvNet.
+
+ Args:
+ num_actions: int, number of actions.
+ quantile_embedding_dim: int, embedding dimension for the quantile input.
+ network_type: namedtuple, collection of expected values to return.
+ state: `tf.Tensor`, contains the agent's current state.
+ num_quantiles: int, number of quantile inputs.
+
+ Returns:
+ net: _network_type object containing the tensors output by the network.
+ """
+ weights_initializer = slim.variance_scaling_initializer(
+ factor=1.0 / np.sqrt(3.0), mode='FAN_IN', uniform=True)
+
+ state_net = tf.cast(state, tf.float32)
+ state_net = tf.div(state_net, 255.)
+ state_net = slim.conv2d(
+ state_net, 32, [8, 8], stride=4,
+ weights_initializer=weights_initializer)
+ state_net = slim.conv2d(
+ state_net, 64, [4, 4], stride=2,
+ weights_initializer=weights_initializer)
+ state_net = slim.conv2d(
+ state_net, 64, [3, 3], stride=1,
+ weights_initializer=weights_initializer)
+ state_net = slim.flatten(state_net)
+ state_net_size = state_net.get_shape().as_list()[-1]
+ state_net_tiled = tf.tile(state_net, [num_quantiles, 1])
+
+ batch_size = state_net.get_shape().as_list()[0]
+ quantiles_shape = [num_quantiles * batch_size, 1]
+ quantiles = tf.random_uniform(
+ quantiles_shape, minval=0, maxval=1, dtype=tf.float32)
+
+ quantile_net = tf.tile(quantiles, [1, quantile_embedding_dim])
+ pi = tf.constant(math.pi)
+ quantile_net = tf.cast(tf.range(
+ 1, quantile_embedding_dim + 1, 1), tf.float32) * pi * quantile_net
+ quantile_net = tf.cos(quantile_net)
+ quantile_net = slim.fully_connected(quantile_net, state_net_size,
+ weights_initializer=weights_initializer)
+ # Hadamard product.
+ net = tf.multiply(state_net_tiled, quantile_net)
+
+ net = slim.fully_connected(
+ net, 512, weights_initializer=weights_initializer)
+ quantile_values = slim.fully_connected(
+ net,
+ num_actions,
+ activation_fn=None,
+ weights_initializer=weights_initializer)
+
+ return network_type(quantile_values=quantile_values, quantiles=quantiles)
+
@gin.configurable
class AtariPreprocessing(object):
diff --git a/dopamine/common/checkpointer.py b/dopamine/discrete_domains/checkpointer.py
similarity index 98%
rename from dopamine/common/checkpointer.py
rename to dopamine/discrete_domains/checkpointer.py
index 08a478af..97a38b35 100644
--- a/dopamine/common/checkpointer.py
+++ b/dopamine/discrete_domains/checkpointer.py
@@ -50,8 +50,7 @@
import os
import pickle
import tensorflow as tf
-
-CHECKPOINT_DURATION = 4
+from dopamine.common import get_checkpoint_duration
def get_latest_checkpoint_number(base_directory):
@@ -141,7 +140,7 @@ def _clean_up_old_checkpoints(self, iteration_number):
# After writing a the checkpoint and sentinel file, we garbage collect files
# that are CHECKPOINT_DURATION * self._checkpoint_frequency versions old.
stale_iteration_number = iteration_number - (self._checkpoint_frequency *
- CHECKPOINT_DURATION)
+ get_checkpoint_duration())
if stale_iteration_number >= 0:
stale_file = self._generate_filename(self._checkpoint_file_prefix,
diff --git a/dopamine/discrete_domains/gym_lib.py b/dopamine/discrete_domains/gym_lib.py
new file mode 100644
index 00000000..01e3dff3
--- /dev/null
+++ b/dopamine/discrete_domains/gym_lib.py
@@ -0,0 +1,335 @@
+# coding=utf-8
+# Copyright 2018 The Dopamine Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Gym-specific (non-Atari) utilities.
+
+Some network specifications specific to certain Gym environments are provided
+here.
+
+Includes a wrapper class around Gym environments. This class makes general Gym
+environments conformant with the API Dopamine is expecting.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import itertools
+import math
+
+
+
+import gym
+import numpy as np
+import tensorflow as tf
+
+import gin.tf
+
+
+CARTPOLE_MIN_VALS = np.array([-2.4, -5., -math.pi/12., -math.pi*2.])
+CARTPOLE_MAX_VALS = np.array([2.4, 5., math.pi/12., math.pi*2.])
+ACROBOT_MIN_VALS = np.array([-1., -1., -1., -1., -5., -5.])
+ACROBOT_MAX_VALS = np.array([1., 1., 1., 1., 5., 5.])
+gin.constant('gym_lib.CARTPOLE_OBSERVATION_SHAPE', (4, 1))
+gin.constant('gym_lib.CARTPOLE_OBSERVATION_DTYPE', tf.float32)
+gin.constant('gym_lib.CARTPOLE_STACK_SIZE', 1)
+gin.constant('gym_lib.ACROBOT_OBSERVATION_SHAPE', (6, 1))
+gin.constant('gym_lib.ACROBOT_OBSERVATION_DTYPE', tf.float32)
+gin.constant('gym_lib.ACROBOT_STACK_SIZE', 1)
+
+slim = tf.contrib.slim
+
+
+@gin.configurable
+def create_gym_environment(environment_name=None, version='v0'):
+ """Wraps a Gym environment with some basic preprocessing.
+
+ Args:
+ environment_name: str, the name of the environment to run.
+ version: str, version of the environment to run.
+
+ Returns:
+ A Gym environment with some standard preprocessing.
+ """
+ assert environment_name is not None
+ full_game_name = '{}-{}'.format(environment_name, version)
+ env = gym.make(full_game_name)
+ # Strip out the TimeLimit wrapper from Gym, which caps us at 200 steps.
+ env = env.env
+ # Wrap the returned environment in a class which conforms to the API expected
+ # by Dopamine.
+ env = GymPreprocessing(env)
+ return env
+
+
+@gin.configurable
+def _basic_discrete_domain_network(min_vals, max_vals, num_actions, state,
+ num_atoms=None):
+ """Builds a basic network for discrete domains, rescaling inputs to [-1, 1].
+
+ Args:
+ min_vals: float, minimum attainable values (must be same shape as `state`).
+ max_vals: float, maximum attainable values (must be same shape as `state`).
+ num_actions: int, number of actions.
+ state: `tf.Tensor`, the state input.
+ num_atoms: int or None, if None will construct a DQN-style network,
+ otherwise will construct a Rainbow-style network.
+
+ Returns:
+ The Q-values for DQN-style agents or logits for Rainbow-style agents.
+ """
+ net = tf.cast(state, tf.float32)
+ net = slim.flatten(net)
+ net -= min_vals
+ net /= max_vals - min_vals
+ net = 2.0 * net - 1.0 # Rescale in range [-1, 1].
+ net = slim.fully_connected(net, 512)
+ net = slim.fully_connected(net, 512)
+ if num_atoms is None:
+ # We are constructing a DQN-style network.
+ return slim.fully_connected(net, num_actions, activation_fn=None)
+ else:
+ # We are constructing a rainbow-style network.
+ return slim.fully_connected(net, num_actions * num_atoms,
+ activation_fn=None)
+
+
+@gin.configurable
+def cartpole_dqn_network(num_actions, network_type, state):
+ """Builds the deep network used to compute the agent's Q-values.
+
+ It rescales the input features to a range that yields improved performance.
+
+ Args:
+ num_actions: int, number of actions.
+ network_type: namedtuple, collection of expected values to return.
+ state: `tf.Tensor`, contains the agent's current state.
+
+ Returns:
+ net: _network_type object containing the tensors output by the network.
+ """
+ q_values = _basic_discrete_domain_network(
+ CARTPOLE_MIN_VALS, CARTPOLE_MAX_VALS, num_actions, state)
+ return network_type(q_values)
+
+
+class FourierBasis(object):
+ """Fourier Basis linear function approximation.
+
+ Requires the ranges for each dimension, and is thus able to use only sine or
+ cosine (and uses cosine). So, this has half the coefficients that a full
+ Fourier approximation would use.
+
+ Many thanks to Will Dabney (wdabney@) for this implementation.
+
+ From the paper:
+ G.D. Konidaris, S. Osentoski and P.S. Thomas. (2011)
+ Value Function Approximation in Reinforcement Learning using the Fourier Basis
+ """
+
+ def __init__(self, nvars, min_vals=0, max_vals=None, order=3):
+ self.order = order
+ self.min_vals = min_vals
+ self.max_vals = max_vals
+ terms = itertools.product(range(order + 1), repeat=nvars)
+
+ # Removing first iterate because it corresponds to the constant bias
+ self.multipliers = tf.constant(
+ [list(map(int, x)) for x in terms][1:], dtype=tf.float32)
+
+ def scale(self, values):
+ shifted = values - self.min_vals
+ if self.max_vals is None:
+ return shifted
+
+ return shifted / (self.max_vals - self.min_vals)
+
+ def compute_features(self, features):
+ # Important to rescale features to be between [0,1]
+ scaled = self.scale(features)
+ return tf.cos(np.pi * tf.matmul(scaled, self.multipliers, transpose_b=True))
+
+
+@gin.configurable
+def fourier_dqn_network(min_vals,
+ max_vals,
+ num_actions,
+ state,
+ fourier_basis_order=3):
+ """Builds the function approximator used to compute the agent's Q-values.
+
+ It uses FourierBasis features and a linear layer.
+
+ Args:
+ min_vals: float, minimum attainable values (must be same shape as `state`).
+ max_vals: float, maximum attainable values (must be same shape as `state`).
+ num_actions: int, number of actions.
+ state: `tf.Tensor`, contains the agent's current state.
+ fourier_basis_order: int, order of the Fourier basis functions.
+
+ Returns:
+ The Q-values for DQN-style agents or logits for Rainbow-style agents.
+ """
+ net = tf.cast(state, tf.float32)
+ net = slim.flatten(net)
+
+ # Feed state through Fourier basis.
+ feature_generator = FourierBasis(
+ net.get_shape().as_list()[-1],
+ min_vals,
+ max_vals,
+ order=fourier_basis_order)
+ net = feature_generator.compute_features(net)
+
+ # Q-values are always linear w.r.t. last layer.
+ q_values = slim.fully_connected(
+ net, num_actions, activation_fn=None, biases_initializer=None)
+ return q_values
+
+
+def cartpole_fourier_dqn_network(num_actions, network_type, state):
+ """Builds the function approximator used to compute the agent's Q-values.
+
+ It uses the Fourier basis features and a linear function approximator.
+
+ Args:
+ num_actions: int, number of actions.
+ network_type: namedtuple, collection of expected values to return.
+ state: `tf.Tensor`, contains the agent's current state.
+
+ Returns:
+ net: _network_type object containing the tensors output by the network.
+ """
+ q_values = fourier_dqn_network(CARTPOLE_MIN_VALS, CARTPOLE_MAX_VALS,
+ num_actions, state)
+ return network_type(q_values)
+
+
+@gin.configurable
+def cartpole_rainbow_network(num_actions, num_atoms, support, network_type,
+ state):
+ """Build the deep network used to compute the agent's Q-value distributions.
+
+ Args:
+ num_actions: int, number of actions.
+ num_atoms: int, the number of buckets of the value function distribution.
+ support: tf.linspace, the support of the Q-value distribution.
+ network_type: `namedtuple`, collection of expected values to return.
+ state: `tf.Tensor`, contains the agent's current state.
+
+ Returns:
+ net: _network_type object containing the tensors output by the network.
+ """
+ net = _basic_discrete_domain_network(
+ CARTPOLE_MIN_VALS, CARTPOLE_MAX_VALS, num_actions, state,
+ num_atoms=num_atoms)
+ logits = tf.reshape(net, [-1, num_actions, num_atoms])
+ probabilities = tf.contrib.layers.softmax(logits)
+ q_values = tf.reduce_sum(support * probabilities, axis=2)
+ return network_type(q_values, logits, probabilities)
+
+
+@gin.configurable
+def acrobot_dqn_network(num_actions, network_type, state):
+ """Builds the deep network used to compute the agent's Q-values.
+
+ It rescales the input features to a range that yields improved performance.
+
+ Args:
+ num_actions: int, number of actions.
+ network_type: namedtuple, collection of expected values to return.
+ state: `tf.Tensor`, contains the agent's current state.
+
+ Returns:
+ net: _network_type object containing the tensors output by the network.
+ """
+ q_values = _basic_discrete_domain_network(
+ ACROBOT_MIN_VALS, ACROBOT_MAX_VALS, num_actions, state)
+ return network_type(q_values)
+
+
+@gin.configurable
+def acrobot_fourier_dqn_network(num_actions, network_type, state):
+ """Builds the function approximator used to compute the agent's Q-values.
+
+ It uses the Fourier basis features and a linear function approximator.
+
+ Args:
+ num_actions: int, number of actions.
+ network_type: namedtuple, collection of expected values to return.
+ state: `tf.Tensor`, contains the agent's current state.
+
+ Returns:
+ net: _network_type object containing the tensors output by the network.
+ """
+ q_values = fourier_dqn_network(ACROBOT_MIN_VALS, ACROBOT_MAX_VALS,
+ num_actions, state)
+ return network_type(q_values)
+
+
+@gin.configurable
+def acrobot_rainbow_network(num_actions, num_atoms, support, network_type,
+ state):
+ """Build the deep network used to compute the agent's Q-value distributions.
+
+ Args:
+ num_actions: int, number of actions.
+ num_atoms: int, the number of buckets of the value function distribution.
+ support: tf.linspace, the support of the Q-value distribution.
+ network_type: `namedtuple`, collection of expected values to return.
+ state: `tf.Tensor`, contains the agent's current state.
+
+ Returns:
+ net: _network_type object containing the tensors output by the network.
+ """
+ net = _basic_discrete_domain_network(
+ ACROBOT_MIN_VALS, ACROBOT_MAX_VALS, num_actions, state,
+ num_atoms=num_atoms)
+ logits = tf.reshape(net, [-1, num_actions, num_atoms])
+ probabilities = tf.contrib.layers.softmax(logits)
+ q_values = tf.reduce_sum(support * probabilities, axis=2)
+ return network_type(q_values, logits, probabilities)
+
+
+@gin.configurable
+class GymPreprocessing(object):
+ """A Wrapper class around Gym environments."""
+
+ def __init__(self, environment):
+ self.environment = environment
+ self.game_over = False
+
+ @property
+ def observation_space(self):
+ return self.environment.observation_space
+
+ @property
+ def action_space(self):
+ return self.environment.action_space
+
+ @property
+ def reward_range(self):
+ return self.environment.reward_range
+
+ @property
+ def metadata(self):
+ return self.environment.metadata
+
+ def reset(self):
+ return self.environment.reset()
+
+ def step(self, action):
+ observation, reward, game_over, info = self.environment.step(action)
+ self.game_over = game_over
+ return observation, reward, game_over, info
diff --git a/dopamine/common/iteration_statistics.py b/dopamine/discrete_domains/iteration_statistics.py
similarity index 100%
rename from dopamine/common/iteration_statistics.py
rename to dopamine/discrete_domains/iteration_statistics.py
diff --git a/dopamine/common/logger.py b/dopamine/discrete_domains/logger.py
similarity index 96%
rename from dopamine/common/logger.py
rename to dopamine/discrete_domains/logger.py
index 8e1b51b2..7bb1ad88 100644
--- a/dopamine/common/logger.py
+++ b/dopamine/discrete_domains/logger.py
@@ -21,9 +21,7 @@
import os
import pickle
import tensorflow as tf
-
-
-CHECKPOINT_DURATION = 4
+from dopamine.common import get_checkpoint_duration
class Logger(object):
@@ -90,7 +88,7 @@ def log_to_file(self, filename_prefix, iteration_number):
pickle.dump(self.data, fout, protocol=pickle.HIGHEST_PROTOCOL)
# After writing a checkpoint file, we garbage collect the log file
# that is CHECKPOINT_DURATION versions old.
- stale_iteration_number = iteration_number - CHECKPOINT_DURATION
+ stale_iteration_number = iteration_number - get_checkpoint_duration()
if stale_iteration_number >= 0:
stale_file = self._generate_filename(filename_prefix,
stale_iteration_number)
diff --git a/dopamine/atari/run_experiment.py b/dopamine/discrete_domains/run_experiment.py
similarity index 82%
rename from dopamine/atari/run_experiment.py
rename to dopamine/discrete_domains/run_experiment.py
index 5a92a202..35c6dda8 100644
--- a/dopamine/atari/run_experiment.py
+++ b/dopamine/discrete_domains/run_experiment.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""Module defining classes and helper methods for running Atari 2600 agents."""
+"""Module defining classes and helper methods for general agents."""
from __future__ import absolute_import
from __future__ import division
@@ -22,21 +22,20 @@
import sys
import time
+from dopamine.agents.dqn import dqn_agent
+from dopamine.agents.implicit_quantile import implicit_quantile_agent
+from dopamine.agents.rainbow import rainbow_agent
+from dopamine.discrete_domains import atari_lib
+from dopamine.discrete_domains import checkpointer
+from dopamine.discrete_domains import iteration_statistics
+from dopamine.discrete_domains import logger
-import atari_py
-from dopamine.atari import preprocessing
-from dopamine.common import checkpointer
-from dopamine.common import iteration_statistics
-from dopamine.common import logger
-import gym
import numpy as np
import tensorflow as tf
import gin.tf
-
-
def load_gin_configs(gin_files, gin_bindings):
"""Loads gin configuration files.
@@ -51,44 +50,73 @@ def load_gin_configs(gin_files, gin_bindings):
skip_unknown=False)
-def create_atari_environment(game_name, sticky_actions=True):
- """Wraps an Atari 2600 Gym environment with some basic preprocessing.
+@gin.configurable
+def create_agent(sess, environment, agent_name=None, summary_writer=None,
+ debug_mode=False):
+ """Creates an agent.
+
+ Args:
+ sess: A `tf.Session` object for running associated ops.
+ environment: An Atari 2600 Gym environment.
+ agent_name: str, name of the agent to create.
+ summary_writer: A Tensorflow summary writer to pass to the agent
+ for in-agent training statistics in Tensorboard.
+ debug_mode: bool, whether to output Tensorboard summaries. If set to true,
+ the agent will output in-episode statistics to Tensorboard. Disabled by
+ default as this results in slower training.
+
+ Returns:
+ agent: An RL agent.
- This preprocessing matches the guidelines proposed in Machado et al. (2017),
- "Revisiting the Arcade Learning Environment: Evaluation Protocols and Open
- Problems for General Agents".
+ Raises:
+ ValueError: If `agent_name` is not in supported list.
+ """
+ assert agent_name is not None
+ if not debug_mode:
+ summary_writer = None
+ if agent_name == 'dqn':
+ return dqn_agent.DQNAgent(sess, num_actions=environment.action_space.n,
+ summary_writer=summary_writer)
+ elif agent_name == 'rainbow':
+ return rainbow_agent.RainbowAgent(
+ sess, num_actions=environment.action_space.n,
+ summary_writer=summary_writer)
+ elif agent_name == 'implicit_quantile':
+ return implicit_quantile_agent.ImplicitQuantileAgent(
+ sess, num_actions=environment.action_space.n,
+ summary_writer=summary_writer)
+ else:
+ raise ValueError('Unknown agent: {}'.format(agent_name))
- The created environment is the Gym wrapper around the Arcade Learning
- Environment.
- The main choice available to the user is whether to use sticky actions or not.
- Sticky actions, as prescribed by Machado et al., cause actions to persist
- with some probability (0.25) when a new command is sent to the ALE. This
- can be viewed as introducing a mild form of stochasticity in the environment.
- We use them by default.
+@gin.configurable
+def create_runner(base_dir, schedule='continuous_train_and_eval'):
+ """Creates an experiment Runner.
Args:
- game_name: str, the name of the Atari 2600 domain.
- sticky_actions: bool, whether to use sticky_actions as per Machado et al.
+ base_dir: str, base directory for hosting all subdirectories.
+ schedule: string, which type of Runner to use.
Returns:
- An Atari 2600 environment with some standard preprocessing.
+ runner: A `Runner` like object.
+
+ Raises:
+ ValueError: When an unknown schedule is encountered.
"""
- game_version = 'v0' if sticky_actions else 'v4'
- full_game_name = '{}NoFrameskip-{}'.format(game_name, game_version)
- env = gym.make(full_game_name)
- # Strip out the TimeLimit wrapper from Gym, which caps us at 100k frames. We
- # handle this time limit internally instead, which lets us cap at 108k frames
- # (30 minutes). The TimeLimit wrapper also plays poorly with saving and
- # restoring states.
- env = env.env
- env = preprocessing.AtariPreprocessing(env)
- return env
+ assert base_dir is not None
+ # Continuously runs training and evaluation until max num_iterations is hit.
+ if schedule == 'continuous_train_and_eval':
+ return Runner(base_dir, create_agent)
+ # Continuously runs training until max num_iterations is hit.
+ elif schedule == 'continuous_train':
+ return TrainRunner(base_dir, create_agent)
+ else:
+ raise ValueError('Unknown schedule: {}'.format(schedule))
@gin.configurable
class Runner(object):
- """Object that handles running Atari 2600 experiments.
+ """Object that handles running Dopamine experiments.
Here we use the term 'experiment' to mean simulating interactions between the
agent and the environment and reporting some statistics pertaining to these
@@ -97,10 +125,11 @@ class Runner(object):
A simple scenario to train a DQN agent is as follows:
```python
+ import dopamine.discrete_domains.atari_lib
base_dir = '/tmp/simple_example'
def create_agent(sess, environment):
return dqn_agent.DQNAgent(sess, num_actions=environment.action_space.n)
- runner = Runner(base_dir, create_agent, game_name='Pong')
+ runner = Runner(base_dir, create_agent, atari_lib.create_atari_environment)
runner.run()
```
"""
@@ -108,9 +137,7 @@ def create_agent(sess, environment):
def __init__(self,
base_dir,
create_agent_fn,
- create_environment_fn=create_atari_environment,
- game_name=None,
- sticky_actions=True,
+ create_environment_fn=atari_lib.create_atari_environment,
checkpoint_file_prefix='ckpt',
logging_file_prefix='log',
log_every_n=1,
@@ -123,11 +150,9 @@ def __init__(self,
Args:
base_dir: str, the base directory to host all required sub-directories.
create_agent_fn: A function that takes as args a Tensorflow session and an
- Atari 2600 Gym environment, and returns an agent.
- create_environment_fn: A function which receives a game name and creates
- an Atari 2600 Gym environment.
- game_name: str, name of the Atari 2600 domain to run.
- sticky_actions: bool, whether to enable sticky actions in the environment.
+ environment, and returns an agent.
+ create_environment_fn: A function which receives a problem name and
+ creates a Gym environment for that problem (e.g. an Atari 2600 game).
checkpoint_file_prefix: str, the prefix to use for checkpoint files.
logging_file_prefix: str, prefix to use for the log files.
log_every_n: int, the frequency for writing logs.
@@ -147,7 +172,6 @@ def __init__(self,
Checkpointer object.
"""
assert base_dir is not None
- assert game_name is not None
self._logging_file_prefix = logging_file_prefix
self._log_every_n = log_every_n
self._num_iterations = num_iterations
@@ -158,7 +182,7 @@ def __init__(self,
self._create_directories()
self._summary_writer = tf.summary.FileWriter(self._base_dir)
- self._environment = create_environment_fn(game_name, sticky_actions)
+ self._environment = create_environment_fn()
# Set up a session and initialize variables.
self._sess = tf.Session('',
config=tf.ConfigProto(allow_soft_placement=True))
@@ -457,24 +481,27 @@ def run_experiment(self):
@gin.configurable
class TrainRunner(Runner):
- """Object that handles running Atari 2600 experiments.
+ """Object that handles running experiments.
The `TrainRunner` differs from the base `Runner` class in that it does not
the evaluation phase. Checkpointing and logging for the train phase are
preserved as before.
"""
- def __init__(self, base_dir, create_agent_fn):
+ def __init__(self, base_dir, create_agent_fn,
+ create_environment_fn=atari_lib.create_atari_environment):
"""Initialize the TrainRunner object in charge of running a full experiment.
Args:
base_dir: str, the base directory to host all required sub-directories.
create_agent_fn: A function that takes as args a Tensorflow session and an
- Atari 2600 Gym environment, and returns an agent.
+ environment, and returns an agent.
+ create_environment_fn: A function which receives a problem name and
+ creates a Gym environment for that problem (e.g. an Atari 2600 game).
"""
tf.logging.info('Creating TrainRunner ...')
- super(TrainRunner, self).__init__(
- base_dir=base_dir, create_agent_fn=create_agent_fn)
+ super(TrainRunner, self).__init__(base_dir, create_agent_fn,
+ create_environment_fn)
self._agent.eval_mode = False
def _run_one_iteration(self, iteration):
diff --git a/dopamine/discrete_domains/train.py b/dopamine/discrete_domains/train.py
new file mode 100644
index 00000000..41c92299
--- /dev/null
+++ b/dopamine/discrete_domains/train.py
@@ -0,0 +1,61 @@
+# coding=utf-8
+# Copyright 2018 The Dopamine Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+r"""The entry point for running a Dopamine agent.
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+
+from absl import app
+from absl import flags
+
+from dopamine.discrete_domains import run_experiment
+
+import tensorflow as tf
+
+
+flags.DEFINE_string('base_dir', None,
+ 'Base directory to host all required sub-directories.')
+flags.DEFINE_multi_string(
+ 'gin_files', [], 'List of paths to gin configuration files (e.g.'
+ '"dopamine/agents/dqn/dqn.gin").')
+flags.DEFINE_multi_string(
+ 'gin_bindings', [],
+ 'Gin bindings to override the values set in the config files '
+ '(e.g. "DQNAgent.epsilon_train=0.1",'
+ ' "create_environment.game_name="Pong"").')
+
+FLAGS = flags.FLAGS
+
+
+def main(unused_argv):
+ """Main method.
+
+ Args:
+ unused_argv: Arguments (unused).
+ """
+ tf.logging.set_verbosity(tf.logging.INFO)
+ run_experiment.load_gin_configs(FLAGS.gin_files, FLAGS.gin_bindings)
+ runner = run_experiment.create_runner(FLAGS.base_dir)
+ runner.run_experiment()
+
+
+if __name__ == '__main__':
+ flags.mark_flag_as_required('base_dir')
+ app.run(main)
diff --git a/dopamine/replay_memory/circular_replay_buffer.py b/dopamine/replay_memory/circular_replay_buffer.py
index e10c03be..f440d19b 100644
--- a/dopamine/replay_memory/circular_replay_buffer.py
+++ b/dopamine/replay_memory/circular_replay_buffer.py
@@ -32,6 +32,7 @@
import numpy as np
import tensorflow as tf
+from dopamine.common import get_checkpoint_duration
import gin.tf
@@ -47,7 +48,6 @@
STORE_FILENAME_PREFIX = '$store$_'
# This constant determines how many iterations a checkpoint is kept for.
-CHECKPOINT_DURATION = 4
MAX_SAMPLE_ATTEMPTS = 1000
@@ -92,6 +92,8 @@ class OutOfGraphReplayBuffer(object):
Attributes:
add_count: int, counter of how many transitions have been added (including
the blank ones at the beginning of an episode).
+ invalid_range: np.array, an array with the indices of cursor-related invalid
+ transitions
"""
def __init__(self,
@@ -103,7 +105,11 @@ def __init__(self,
gamma=0.99,
max_sample_attempts=MAX_SAMPLE_ATTEMPTS,
extra_storage_types=None,
- observation_dtype=np.uint8):
+ observation_dtype=np.uint8,
+ action_shape=(),
+ action_dtype=np.int32,
+ reward_shape=(),
+ reward_dtype=np.float32):
"""Initializes OutOfGraphReplayBuffer.
Args:
@@ -119,6 +125,12 @@ def __init__(self,
contents that will be stored and returned by sample_transition_batch.
observation_dtype: np.dtype, type of the observations. Defaults to
np.uint8 for Atari 2600.
+ action_shape: tuple of ints, the shape for the action vector. Empty tuple
+ means the action is a scalar.
+ action_dtype: np.dtype, type of elements in the action.
+ reward_shape: tuple of ints, the shape of the reward vector. Empty tuple
+ means the reward is a scalar.
+ reward_dtype: np.dtype, type of elements in the reward.
Raises:
ValueError: If replay_capacity is too small to hold at least one
@@ -140,6 +152,10 @@ def __init__(self,
tf.logging.info('\t update_horizon: %d', update_horizon)
tf.logging.info('\t gamma: %f', gamma)
+ self._action_shape = action_shape
+ self._action_dtype = action_dtype
+ self._reward_shape = reward_shape
+ self._reward_dtype = reward_dtype
self._observation_shape = observation_shape
self._stack_size = stack_size
self._state_shape = self._observation_shape + (self._stack_size,)
@@ -193,8 +209,8 @@ def get_storage_signature(self):
storage_elements = [
ReplayElement('observation', self._observation_shape,
self._observation_dtype),
- ReplayElement('action', (), np.int32),
- ReplayElement('reward', (), np.float32),
+ ReplayElement('action', self._action_shape, self._action_dtype),
+ ReplayElement('reward', self._reward_shape, self._reward_dtype),
ReplayElement('terminal', (), np.uint8)
]
@@ -325,7 +341,10 @@ def get_range(self, array, start_index, end_index):
return return_array
def get_observation_stack(self, index):
- state = self.get_range(self._store['observation'],
+ return self._get_element_stack(index, 'observation')
+
+ def _get_element_stack(self, index, element_name):
+ state = self.get_range(self._store[element_name],
index - self._stack_size + 1, index + 1)
# The stacking axis is 0 but the agent expects as the last axis.
return np.moveaxis(state, 0, -1)
@@ -491,12 +510,16 @@ def sample_transition_batch(self, batch_size=None, indices=None):
if element.name == 'state':
element_array[batch_element] = self.get_observation_stack(state_index)
elif element.name == 'reward':
- # cumpute the discounted sum of rewards in the trajectory.
- element_array[batch_element] = trajectory_discount_vector.dot(
- trajectory_rewards)
+ # compute the discounted sum of rewards in the trajectory.
+ element_array[batch_element] = np.sum(
+ trajectory_discount_vector * trajectory_rewards, axis=0)
elif element.name == 'next_state':
element_array[batch_element] = self.get_observation_stack(
(next_state_index) % self._replay_capacity)
+ elif element.name in ('next_action', 'next_reward'):
+ element_array[batch_element] = (
+ self._store[element.name.lstrip('next_')][(next_state_index) %
+ self._replay_capacity])
elif element.name == 'terminal':
element_array[batch_element] = is_terminal_transition
elif element.name == 'indices':
@@ -522,10 +545,16 @@ def get_transition_elements(self, batch_size=None):
transition_elements = [
ReplayElement('state', (batch_size,) + self._state_shape,
self._observation_dtype),
- ReplayElement('action', (batch_size,), np.int32),
- ReplayElement('reward', (batch_size,), np.float32),
+ ReplayElement('action', (batch_size,) + self._action_shape,
+ self._action_dtype),
+ ReplayElement('reward', (batch_size,) + self._reward_shape,
+ self._reward_dtype),
ReplayElement('next_state', (batch_size,) + self._state_shape,
self._observation_dtype),
+ ReplayElement('next_action', (batch_size,) + self._action_shape,
+ self._action_dtype),
+ ReplayElement('next_reward', (batch_size,) + self._reward_shape,
+ self._reward_dtype),
ReplayElement('terminal', (batch_size,), np.uint8),
ReplayElement('indices', (batch_size,), np.int32)
]
@@ -589,7 +618,7 @@ def save(self, checkpoint_dir, iteration_number):
# After writing a checkpoint file, we garbage collect the checkpoint file
# that is four versions old.
- stale_iteration_number = iteration_number - CHECKPOINT_DURATION
+ stale_iteration_number = iteration_number - get_checkpoint_duration()
if stale_iteration_number >= 0:
stale_filename = self._generate_filename(checkpoint_dir, attr,
stale_iteration_number)
@@ -657,7 +686,11 @@ def __init__(self,
wrapped_memory=None,
max_sample_attempts=MAX_SAMPLE_ATTEMPTS,
extra_storage_types=None,
- observation_dtype=np.uint8):
+ observation_dtype=np.uint8,
+ action_shape=(),
+ action_dtype=np.int32,
+ reward_shape=(),
+ reward_dtype=np.float32):
"""Initializes WrappedReplayBuffer.
Args:
@@ -677,6 +710,12 @@ def __init__(self,
contents that will be stored and returned by sample_transition_batch.
observation_dtype: np.dtype, type of the observations. Defaults to
np.uint8 for Atari 2600.
+ action_shape: tuple of ints, the shape for the action vector. Empty tuple
+ means the action is a scalar.
+ action_dtype: np.dtype, type of elements in the action.
+ reward_shape: tuple of ints, the shape of the reward vector. Empty tuple
+ means the reward is a scalar.
+ reward_dtype: np.dtype, type of elements in the reward.
Raises:
ValueError: If update_horizon is not positive.
@@ -698,10 +737,19 @@ def __init__(self,
self.memory = wrapped_memory
else:
self.memory = OutOfGraphReplayBuffer(
- observation_shape, stack_size, replay_capacity, batch_size,
- update_horizon, gamma, max_sample_attempts,
+ observation_shape,
+ stack_size,
+ replay_capacity,
+ batch_size,
+ update_horizon,
+ gamma,
+ max_sample_attempts,
observation_dtype=observation_dtype,
- extra_storage_types=extra_storage_types)
+ extra_storage_types=extra_storage_types,
+ action_shape=action_shape,
+ action_dtype=action_dtype,
+ reward_shape=reward_shape,
+ reward_dtype=reward_dtype)
self.create_sampling_ops(use_staging)
@@ -810,6 +858,8 @@ def unpack_transition(self, transition_tensors, transition_type):
self.actions = self.transition['action']
self.rewards = self.transition['reward']
self.next_states = self.transition['next_state']
+ self.next_actions = self.transition['next_action']
+ self.next_rewards = self.transition['next_reward']
self.terminals = self.transition['terminal']
self.indices = self.transition['indices']
diff --git a/dopamine/replay_memory/prioritized_replay_buffer.py b/dopamine/replay_memory/prioritized_replay_buffer.py
index 426cc1b4..449e76b9 100644
--- a/dopamine/replay_memory/prioritized_replay_buffer.py
+++ b/dopamine/replay_memory/prioritized_replay_buffer.py
@@ -49,7 +49,11 @@ def __init__(self,
gamma=0.99,
max_sample_attempts=circular_replay_buffer.MAX_SAMPLE_ATTEMPTS,
extra_storage_types=None,
- observation_dtype=np.uint8):
+ observation_dtype=np.uint8,
+ action_shape=(),
+ action_dtype=np.int32,
+ reward_shape=(),
+ reward_dtype=np.float32):
"""Initializes OutOfGraphPrioritizedReplayBuffer.
Args:
@@ -65,6 +69,12 @@ def __init__(self,
contents that will be stored and returned by sample_transition_batch.
observation_dtype: np.dtype, type of the observations. Defaults to
np.uint8 for Atari 2600.
+ action_shape: tuple of ints, the shape for the action vector. Empty tuple
+ means the action is a scalar.
+ action_dtype: np.dtype, type of elements in the action.
+ reward_shape: tuple of ints, the shape of the reward vector. Empty tuple
+ means the reward is a scalar.
+ reward_dtype: np.dtype, type of elements in the reward.
"""
super(OutOfGraphPrioritizedReplayBuffer, self).__init__(
observation_shape=observation_shape,
@@ -75,7 +85,11 @@ def __init__(self,
gamma=gamma,
max_sample_attempts=max_sample_attempts,
extra_storage_types=extra_storage_types,
- observation_dtype=observation_dtype)
+ observation_dtype=observation_dtype,
+ action_shape=action_shape,
+ action_dtype=action_dtype,
+ reward_shape=reward_shape,
+ reward_dtype=reward_dtype)
self.sum_tree = sum_tree.SumTree(replay_capacity)
@@ -140,7 +154,7 @@ def sample_index_batch(self, batch_size):
if not self.is_valid_transition(indices[i]):
if allowed_attempts == 0:
raise RuntimeError(
- 'Max saple attempsts: Tried {} times but only sampled {}'
+ 'Max sample attempts: Tried {} times but only sampled {}'
' valid indices. Batch size is {}'.
format(self._max_sample_attempts, i, batch_size))
index = indices[i]
@@ -259,7 +273,11 @@ def __init__(self,
gamma=0.99,
max_sample_attempts=circular_replay_buffer.MAX_SAMPLE_ATTEMPTS,
extra_storage_types=None,
- observation_dtype=np.uint8):
+ observation_dtype=np.uint8,
+ action_shape=(),
+ action_dtype=np.int32,
+ reward_shape=(),
+ reward_dtype=np.float32):
"""Initializes WrappedPrioritizedReplayBuffer.
Args:
@@ -277,6 +295,12 @@ def __init__(self,
contents that will be stored and returned by sample_transition_batch.
observation_dtype: np.dtype, type of the observations. Defaults to
np.uint8 for Atari 2600.
+ action_shape: tuple of ints, the shape for the action vector. Empty tuple
+ means the action is a scalar.
+ action_dtype: np.dtype, type of elements in the action.
+ reward_shape: tuple of ints, the shape of the reward vector. Empty tuple
+ means the reward is a scalar.
+ reward_dtype: np.dtype, type of elements in the reward.
Raises:
ValueError: If update_horizon is not positive.
@@ -285,7 +309,8 @@ def __init__(self,
memory = OutOfGraphPrioritizedReplayBuffer(
observation_shape, stack_size, replay_capacity, batch_size,
update_horizon, gamma, max_sample_attempts,
- extra_storage_types=extra_storage_types)
+ extra_storage_types=extra_storage_types,
+ observation_dtype=observation_dtype)
super(WrappedPrioritizedReplayBuffer, self).__init__(
observation_shape,
stack_size,
@@ -295,7 +320,12 @@ def __init__(self,
update_horizon,
gamma,
wrapped_memory=memory,
- extra_storage_types=extra_storage_types)
+ extra_storage_types=extra_storage_types,
+ observation_dtype=observation_dtype,
+ action_shape=action_shape,
+ action_dtype=action_dtype,
+ reward_shape=reward_shape,
+ reward_dtype=reward_dtype)
def tf_set_priority(self, indices, priorities):
"""Sets the priorities for the given indices.
diff --git a/gym/preprocessing.py b/gym/preprocessing.py
deleted file mode 100644
index 80f7233e..00000000
--- a/gym/preprocessing.py
+++ /dev/null
@@ -1,54 +0,0 @@
-# coding=utf-8
-# Copyright 2018 The Dopamine Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""A wrapper class around Gym environments.
-
-This class makes general Gym environments conformant with the API Dopamine is
-expecting.
-"""
-
-import gin.tf
-
-
-@gin.configurable
-class GymPreprocessing(object):
- """A Wrapper class around Gym environments."""
-
- def __init__(self, environment):
- self.environment = environment
- self.game_over = False
-
- @property
- def observation_space(self):
- return self.environment.observation_space
-
- @property
- def action_space(self):
- return self.environment.action_space
-
- @property
- def reward_range(self):
- return self.environment.reward_range
-
- @property
- def metadata(self):
- return self.environment.metadata
-
- def reset(self):
- return self.environment.reset()
-
- def step(self, action):
- observation, reward, game_over, info = self.environment.step(action)
- self.game_over = game_over
- return observation, reward, game_over, info
diff --git a/setup.py b/setup.py
index adb82f4a..c147c9c9 100644
--- a/setup.py
+++ b/setup.py
@@ -43,7 +43,7 @@
setup(
name='dopamine_rl',
- version='1.0.5',
+ version='2.0.0',
include_package_data=True,
packages=find_packages(exclude=['docs']), # Required
package_data={'testdata': ['testdata/*.gin']},
diff --git a/tests/dopamine/atari/train_test.py b/tests/dopamine/atari/train_test.py
deleted file mode 100644
index 2f13c95f..00000000
--- a/tests/dopamine/atari/train_test.py
+++ /dev/null
@@ -1,116 +0,0 @@
-# coding=utf-8
-# Copyright 2018 The Dopamine Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Tests for dopamine.atari.train."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-
-
-from absl import flags
-from absl.testing import flagsaver
-from dopamine.atari import run_experiment
-from dopamine.atari import train
-import mock
-import tensorflow as tf
-
-import gin.tf
-
-FLAGS = flags.FLAGS
-
-
-class TrainTest(tf.test.TestCase):
-
- def testCreateDQNAgent(self):
- FLAGS.agent_name = 'dqn'
- with mock.patch.object(train, 'dqn_agent') as mock_dqn_agent:
-
- def mock_fn(unused_sess, num_actions, summary_writer):
- del summary_writer
- return num_actions * 10
-
- mock_dqn_agent.DQNAgent.side_effect = mock_fn
- environment = mock.Mock()
- environment.action_space.n = 7
- self.assertEqual(70, train.create_agent(self.test_session(), environment))
-
- def testCreateRainbowAgent(self):
- FLAGS.agent_name = 'rainbow'
- with mock.patch.object(train, 'rainbow_agent') as mock_rainbow_agent:
-
- def mock_fn(unused_sess, num_actions, summary_writer):
- del summary_writer
- return num_actions * 10
-
- mock_rainbow_agent.RainbowAgent.side_effect = mock_fn
- environment = mock.Mock()
- environment.action_space.n = 7
- self.assertEqual(70, train.create_agent(self.test_session(), environment))
-
- @mock.patch.object(run_experiment, 'Runner')
- def testCreateRunnerUnknown(self, mock_runner_constructor):
- mock_create_agent = mock.Mock()
- base_dir = '/tmp'
- FLAGS.schedule = 'unknown_schedule'
- with self.assertRaisesRegexp(ValueError, 'Unknown schedule'):
- train.create_runner(base_dir, mock_create_agent)
-
- @mock.patch.object(run_experiment, 'Runner')
- def testCreateRunner(self, mock_runner_constructor):
- mock_create_agent = mock.Mock()
- base_dir = '/tmp'
- train.create_runner(base_dir, mock_create_agent)
- self.assertEqual(1, mock_runner_constructor.call_count)
- mock_args, _ = mock_runner_constructor.call_args
- self.assertEqual(base_dir, mock_args[0])
- self.assertEqual(mock_create_agent, mock_args[1])
-
- @flagsaver.flagsaver(schedule='continuous_train')
- @mock.patch.object(run_experiment, 'TrainRunner')
- def testCreateTrainRunner(self, mock_runner_constructor):
- mock_create_agent = mock.Mock()
- base_dir = '/tmp'
- train.create_runner(base_dir, mock_create_agent)
- self.assertEqual(1, mock_runner_constructor.call_count)
- mock_args, _ = mock_runner_constructor.call_args
- self.assertEqual(base_dir, mock_args[0])
- self.assertEqual(mock_create_agent, mock_args[1])
-
- @flagsaver.flagsaver(gin_files=['file1', 'file2', 'file3'])
- @flagsaver.flagsaver(gin_bindings=['binding1', 'binding2'])
- @mock.patch.object(gin, 'parse_config_files_and_bindings')
- @mock.patch.object(run_experiment, 'Runner')
- def testLaunchExperiment(
- self, mock_runner_constructor, mock_parse_config_files_and_bindings):
- mock_create_agent = mock.Mock()
- mock_runner = mock.Mock()
- mock_runner_constructor.return_value = mock_runner
-
- def mock_create_runner(unused_base_dir, unused_create_agent_fn):
- return mock_runner
-
- train.launch_experiment(mock_create_runner, mock_create_agent)
- self.assertEqual(1, mock_parse_config_files_and_bindings.call_count)
- mock_args, mock_kwargs = mock_parse_config_files_and_bindings.call_args
- self.assertEqual(FLAGS.gin_files, mock_args[0])
- self.assertEqual(FLAGS.gin_bindings, mock_kwargs['bindings'])
- self.assertFalse(mock_kwargs['skip_unknown'])
- self.assertEqual(1, mock_runner.run_experiment.call_count)
-
-
-if __name__ == '__main__':
- tf.test.main()
diff --git a/tests/dopamine/atari_init_test.py b/tests/dopamine/atari_init_test.py
index 640bf004..0b7d147c 100644
--- a/tests/dopamine/atari_init_test.py
+++ b/tests/dopamine/atari_init_test.py
@@ -21,7 +21,7 @@
from absl import flags
-from dopamine.atari import train
+from dopamine.discrete_domains import train
import tensorflow as tf
@@ -34,7 +34,6 @@ def setUp(self):
FLAGS.base_dir = os.path.join(
'/tmp/dopamine_tests',
datetime.datetime.utcnow().strftime('run_%Y_%m_%d_%H_%M_%S'))
- FLAGS.agent_name = 'dqn'
FLAGS.gin_files = ['dopamine/agents/dqn/configs/dqn.gin']
# `num_iterations` set to zero to prevent runner execution.
FLAGS.gin_bindings = [
diff --git a/tests/dopamine/atari/preprocessing_test.py b/tests/dopamine/discrete_domains/atari_lib_test.py
similarity index 70%
rename from tests/dopamine/atari/preprocessing_test.py
rename to tests/dopamine/discrete_domains/atari_lib_test.py
index 4d66a9d8..a110fda3 100644
--- a/tests/dopamine/atari/preprocessing_test.py
+++ b/tests/dopamine/discrete_domains/atari_lib_test.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""Tests for dopamine.atari.preprocessing."""
+"""Tests for dopamine.discrete_domains.atari_lib."""
from __future__ import absolute_import
from __future__ import division
@@ -20,14 +20,44 @@
+
from absl import flags
-from dopamine.atari import preprocessing
+from dopamine.discrete_domains import atari_lib
+import gym
+import mock
import numpy as np
import tensorflow as tf
FLAGS = flags.FLAGS
+class AtariLibTest(tf.test.TestCase):
+
+
+ def testCreateAtariEnvironmentWithoutGameName(self):
+ with self.assertRaises(AssertionError):
+ atari_lib.create_atari_environment()
+
+ @mock.patch.object(atari_lib, 'AtariPreprocessing')
+ @mock.patch.object(gym, 'make')
+ def testCreateAtariEnvironment(self, mock_gym_make, mock_atari_lib):
+ class MockGymEnv(object):
+
+ def __init__(self, env_name):
+ self.env = 'gym({})'.format(env_name)
+
+ def fake_make_env(name):
+ return MockGymEnv(name)
+
+ mock_gym_make.side_effect = fake_make_env
+ # pylint: disable=unnecessary-lambda
+ mock_atari_lib.side_effect = lambda x: 'atari({})'.format(x)
+ # pylint: enable=unnecessary-lambda
+ game_name = 'Test'
+ env = atari_lib.create_atari_environment(game_name)
+ self.assertEqual('atari(gym(TestNoFrameskip-v0))', env)
+
+
class MockALE(object):
"""Mock internal ALE for testing."""
@@ -77,7 +107,7 @@ class AtariPreprocessingTest(tf.test.TestCase):
def testResetPassesObservation(self):
env = MockEnvironment()
- env = preprocessing.AtariPreprocessing(env, frame_skip=1, screen_size=16)
+ env = atari_lib.AtariPreprocessing(env, frame_skip=1, screen_size=16)
observation = env.reset()
self.assertEqual(observation.shape, (16, 16, 1))
@@ -85,7 +115,7 @@ def testResetPassesObservation(self):
def testTerminalPassedThrough(self):
max_steps = 10
env = MockEnvironment(max_steps=max_steps)
- env = preprocessing.AtariPreprocessing(env, frame_skip=1)
+ env = atari_lib.AtariPreprocessing(env, frame_skip=1)
env.reset()
# Make sure we get the right number of steps.
@@ -99,7 +129,7 @@ def testTerminalPassedThrough(self):
def testFrameSkipAccumulatesReward(self):
frame_skip = 2
env = MockEnvironment()
- env = preprocessing.AtariPreprocessing(env, frame_skip=frame_skip)
+ env = atari_lib.AtariPreprocessing(env, frame_skip=frame_skip)
env.reset()
# Make sure we get the right number of steps. Reward is 1 when we
@@ -110,7 +140,7 @@ def testFrameSkipAccumulatesReward(self):
def testMaxFramePooling(self):
frame_skip = 2
env = MockEnvironment()
- env = preprocessing.AtariPreprocessing(env, frame_skip=frame_skip)
+ env = atari_lib.AtariPreprocessing(env, frame_skip=frame_skip)
env.reset()
# The first observation is 2, the second 0; max is 2.
diff --git a/tests/dopamine/common/checkpointer_test.py b/tests/dopamine/discrete_domains/checkpointer_test.py
similarity index 99%
rename from tests/dopamine/common/checkpointer_test.py
rename to tests/dopamine/discrete_domains/checkpointer_test.py
index 21903f69..03f0e55b 100644
--- a/tests/dopamine/common/checkpointer_test.py
+++ b/tests/dopamine/discrete_domains/checkpointer_test.py
@@ -24,7 +24,7 @@
from absl import flags
-from dopamine.common import checkpointer
+from dopamine.discrete_domains import checkpointer
import tensorflow as tf
FLAGS = flags.FLAGS
diff --git a/tests/gym/preprocessing_test.py b/tests/dopamine/discrete_domains/gym_lib_test.py
similarity index 91%
rename from tests/gym/preprocessing_test.py
rename to tests/dopamine/discrete_domains/gym_lib_test.py
index 3873a92c..ecfec855 100644
--- a/tests/gym/preprocessing_test.py
+++ b/tests/dopamine/discrete_domains/gym_lib_test.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""Tests for dopamine.gym.preprocessing."""
+"""Tests for dopamine.discrete_domains.gym_lib."""
from __future__ import absolute_import
from __future__ import division
@@ -20,7 +20,7 @@
-from dopamine.gym import preprocessing
+from dopamine.discrete_domains import gym_lib
import tensorflow as tf
@@ -43,7 +43,7 @@ def step(self, unused_action):
class GymPreprocessingTest(tf.test.TestCase):
def testAll(self):
- env = preprocessing.GymPreprocessing(MockGymEnvironment())
+ env = gym_lib.GymPreprocessing(MockGymEnvironment())
self.assertEqual('observation_space', env.observation_space)
self.assertEqual('action_space', env.action_space)
self.assertEqual('reward_range', env.reward_range)
diff --git a/tests/dopamine/common/iteration_statistics_test.py b/tests/dopamine/discrete_domains/iteration_statistics_test.py
similarity index 97%
rename from tests/dopamine/common/iteration_statistics_test.py
rename to tests/dopamine/discrete_domains/iteration_statistics_test.py
index 7fe069f2..835264f8 100644
--- a/tests/dopamine/common/iteration_statistics_test.py
+++ b/tests/dopamine/discrete_domains/iteration_statistics_test.py
@@ -20,7 +20,7 @@
-from dopamine.common import iteration_statistics
+from dopamine.discrete_domains import iteration_statistics
import tensorflow as tf
diff --git a/tests/dopamine/common/logger_test.py b/tests/dopamine/discrete_domains/logger_test.py
similarity index 98%
rename from tests/dopamine/common/logger_test.py
rename to tests/dopamine/discrete_domains/logger_test.py
index 82736dc6..c916f810 100644
--- a/tests/dopamine/common/logger_test.py
+++ b/tests/dopamine/discrete_domains/logger_test.py
@@ -24,7 +24,7 @@
from absl import flags
-from dopamine.common import logger
+from dopamine.discrete_domains import logger
import tensorflow as tf
FLAGS = flags.FLAGS
diff --git a/tests/dopamine/atari/run_experiment_test.py b/tests/dopamine/discrete_domains/run_experiment_test.py
similarity index 77%
rename from tests/dopamine/atari/run_experiment_test.py
rename to tests/dopamine/discrete_domains/run_experiment_test.py
index 5fc5a97e..8f130495 100644
--- a/tests/dopamine/atari/run_experiment_test.py
+++ b/tests/dopamine/discrete_domains/run_experiment_test.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""Tests for dopamine.atari.run_experiment."""
+"""Tests for dopamine.common.run_experiment."""
from __future__ import absolute_import
from __future__ import division
@@ -24,9 +24,12 @@
from absl import flags
-from dopamine.atari import run_experiment
-from dopamine.common import checkpointer
-from dopamine.common import logger
+from dopamine.agents.dqn import dqn_agent
+from dopamine.agents.implicit_quantile import implicit_quantile_agent
+from dopamine.agents.rainbow import rainbow_agent
+from dopamine.discrete_domains import checkpointer
+from dopamine.discrete_domains import logger
+from dopamine.discrete_domains import run_experiment
import mock
import tensorflow as tf
@@ -108,6 +111,74 @@ def testLoadGinConfigs(self, mock_parse_config_files_and_bindings):
self.assertEqual(gin_bindings, mock_kwargs['bindings'])
self.assertFalse(mock_kwargs['skip_unknown'])
+ def testNoAgentName(self):
+ with self.assertRaises(AssertionError):
+ _ = run_experiment.create_agent(self.test_session(), mock.Mock())
+
+ @mock.patch.object(dqn_agent, 'DQNAgent')
+ def testCreateDQNAgent(self, mock_dqn_agent):
+ def mock_fn(unused_sess, num_actions, summary_writer):
+ del summary_writer
+ return num_actions * 10
+
+ mock_dqn_agent.side_effect = mock_fn
+ environment = mock.Mock()
+ environment.action_space.n = 7
+ self.assertEqual(70, run_experiment.create_agent(self.test_session(),
+ environment,
+ agent_name='dqn'))
+
+ @mock.patch.object(rainbow_agent, 'RainbowAgent')
+ def testCreateRainbowAgent(self, mock_rainbow_agent):
+ def mock_fn(unused_sess, num_actions, summary_writer):
+ del summary_writer
+ return num_actions * 10
+
+ mock_rainbow_agent.side_effect = mock_fn
+ environment = mock.Mock()
+ environment.action_space.n = 7
+ self.assertEqual(70, run_experiment.create_agent(self.test_session(),
+ environment,
+ agent_name='rainbow'))
+
+ @mock.patch.object(implicit_quantile_agent, 'ImplicitQuantileAgent')
+ def testCreateImplicitQuantileAgent(self, mock_implicit_quantile_agent):
+ def mock_fn(unused_sess, num_actions, summary_writer):
+ del summary_writer
+ return num_actions * 10
+
+ mock_implicit_quantile_agent.side_effect = mock_fn
+ environment = mock.Mock()
+ environment.action_space.n = 7
+ self.assertEqual(70, run_experiment.create_agent(
+ self.test_session(), environment, agent_name='implicit_quantile'))
+
+ def testCreateRunnerUnknown(self):
+ base_dir = '/tmp'
+ with self.assertRaisesRegexp(ValueError, 'Unknown schedule'):
+ run_experiment.create_runner(base_dir,
+ 'Unknown schedule')
+
+ @mock.patch.object(run_experiment, 'Runner')
+ @mock.patch.object(run_experiment, 'create_agent')
+ def testCreateRunner(self, mock_create_agent, mock_runner_constructor):
+ base_dir = '/tmp'
+ run_experiment.create_runner(base_dir)
+ self.assertEqual(1, mock_runner_constructor.call_count)
+ mock_args, _ = mock_runner_constructor.call_args
+ self.assertEqual(base_dir, mock_args[0])
+ self.assertEqual(mock_create_agent, mock_args[1])
+
+ @mock.patch.object(run_experiment, 'TrainRunner')
+ @mock.patch.object(run_experiment, 'create_agent')
+ def testCreateTrainRunner(self, mock_create_agent, mock_runner_constructor):
+ base_dir = '/tmp'
+ run_experiment.create_runner(base_dir,
+ schedule='continuous_train')
+ self.assertEqual(1, mock_runner_constructor.call_count)
+ mock_args, _ = mock_runner_constructor.call_args
+ self.assertEqual(base_dir, mock_args[0])
+ self.assertEqual(mock_create_agent, mock_args[1])
class RunnerTest(tf.test.TestCase):
@@ -129,18 +200,13 @@ def setUp(self):
shutil.rmtree(self._test_subdir, ignore_errors=True)
os.makedirs(self._test_subdir)
- def testFailsWithoutGameName(self):
- with self.assertRaises(AssertionError):
- run_experiment.Runner(self._test_subdir, self._create_agent_fn)
-
@mock.patch.object(checkpointer, 'get_latest_checkpoint_number')
def testInitializeCheckpointingWithNoCheckpointFile(self, mock_get_latest):
mock_get_latest.return_value = -1
base_dir = '/does/not/exist'
with self.assertRaisesRegexp(tf.errors.PermissionDeniedError,
'.*/does.*'):
- run_experiment.Runner(base_dir, self._create_agent_fn,
- game_name='Pong')
+ run_experiment.Runner(base_dir, self._create_agent_fn, mock.Mock)
@mock.patch.object(checkpointer, 'get_latest_checkpoint_number')
@mock.patch.object(checkpointer, 'Checkpointer')
@@ -158,8 +224,7 @@ def testInitializeCheckpointingWhenCheckpointUnbundleFails(
mock_logger_constructor.return_value = mock_logger
runner = run_experiment.Runner(self._test_subdir,
lambda x, y, summary_writer: agent,
- create_environment_fn=lambda x, y: x,
- game_name='Test')
+ mock.Mock)
self.assertEqual(0, runner._start_iteration)
self.assertEqual(1, mock_checkpointer.load_checkpoint.call_count)
self.assertEqual(1, agent.unbundle.call_count)
@@ -186,7 +251,7 @@ def testInitializeCheckpointingWhenCheckpointUnbundleSucceeds(
mock_agent.unbundle.return_value = True
runner = run_experiment.Runner(self._test_subdir,
lambda x, y, summary_writer: mock_agent,
- game_name='Pong')
+ mock.Mock)
expected_iteration = current_iteration + 1
self.assertEqual(expected_iteration, runner._start_iteration)
self.assertDictEqual(logs_data, runner._logger.data)
@@ -197,8 +262,7 @@ def testRunOneEpisode(self):
max_steps_per_episode = 11
environment = MockEnvironment()
runner = run_experiment.Runner(
- self._test_subdir, self._create_agent_fn, game_name='Test',
- create_environment_fn=lambda x, y: environment,
+ self._test_subdir, self._create_agent_fn, lambda: environment,
max_steps_per_episode=max_steps_per_episode)
step_number, total_reward = runner._run_one_episode()
self.assertEqual(self._agent.step.call_count, environment.max_steps - 1)
@@ -211,8 +275,7 @@ def testRunOneEpisodeWithLowMaxSteps(self):
max_steps_per_episode = 2
environment = MockEnvironment()
runner = run_experiment.Runner(
- self._test_subdir, self._create_agent_fn, game_name='Test',
- create_environment_fn=lambda x, y: environment,
+ self._test_subdir, self._create_agent_fn, lambda: environment,
max_steps_per_episode=max_steps_per_episode)
step_number, total_reward = runner._run_one_episode()
self.assertEqual(self._agent.step.call_count, max_steps_per_episode - 1)
@@ -226,8 +289,7 @@ def testRunOnePhase(self):
environment = MockEnvironment(max_steps=environment_steps)
statistics = []
runner = run_experiment.Runner(
- self._test_subdir, self._create_agent_fn, game_name='Test',
- create_environment_fn=lambda x, y: environment)
+ self._test_subdir, self._create_agent_fn, lambda: environment)
step_number, sum_returns, num_episodes = runner._run_one_phase(
max_steps, statistics, 'test')
calls_to_run_episode = int(max_steps / environment_steps)
@@ -252,9 +314,9 @@ def testRunOneIteration(self):
training_steps = 20
evaluation_steps = 10
runner = run_experiment.Runner(
- self._test_subdir, self._create_agent_fn, game_name='Test',
- create_environment_fn=lambda x, y: environment,
- training_steps=training_steps, evaluation_steps=evaluation_steps)
+ self._test_subdir, self._create_agent_fn, lambda: environment,
+ training_steps=training_steps,
+ evaluation_steps=evaluation_steps)
dictionary = runner._run_one_iteration(1)
train_calls = int(training_steps / environment_steps)
eval_calls = int(evaluation_steps / environment_steps)
@@ -276,9 +338,7 @@ def testLogExperiment(self, mock_logger_constructor):
experiment_logger = MockLogger(test_cls=self)
mock_logger_constructor.return_value = experiment_logger
runner = run_experiment.Runner(
- self._test_subdir, self._create_agent_fn,
- game_name='Test',
- create_environment_fn=lambda x, y: mock.Mock(),
+ self._test_subdir, self._create_agent_fn, mock.Mock,
logging_file_prefix=logging_file_prefix,
log_every_n=log_every_n)
num_iterations = 10
@@ -308,9 +368,7 @@ def bundle_and_checkpoint(x, y):
mock_logger = MockLogger(run_asserts=False, data=logs_data)
mock_logger_constructor.return_value = mock_logger
runner = run_experiment.Runner(
- self._test_subdir, self._create_agent_fn,
- game_name='Test',
- create_environment_fn=lambda x, y: mock.Mock())
+ self._test_subdir, self._create_agent_fn, mock.Mock)
runner._checkpoint_experiment(iteration)
self.assertEqual(1, experiment_checkpointer.save_checkpoint.call_count)
mock_args, _ = experiment_checkpointer.save_checkpoint.call_args
@@ -328,9 +386,7 @@ def testRunExperimentWithInconsistentRange(self, mock_logger_constructor,
experiment_checkpointer = mock.Mock()
mock_checkpointer_constructor.return_value = experiment_checkpointer
runner = run_experiment.Runner(
- self._test_subdir, self._create_agent_fn,
- game_name='Test',
- create_environment_fn=lambda x, y: mock.Mock(),
+ self._test_subdir, self._create_agent_fn, mock.Mock,
num_iterations=0)
runner.run_experiment()
self.assertEqual(0, experiment_checkpointer.save_checkpoint.call_count)
@@ -364,9 +420,7 @@ def bundle_and_checkpoint(x, y):
self._agent.unbundle.return_value = True
end_iteration = start_iteration + num_iterations
runner = run_experiment.Runner(
- self._test_subdir, self._create_agent_fn,
- game_name='Test',
- create_environment_fn=lambda x, y: environment,
+ self._test_subdir, self._create_agent_fn, lambda: environment,
log_every_n=log_every_n,
num_iterations=end_iteration,
training_steps=1,
diff --git a/tests/dopamine/replay_memory/circular_replay_buffer_test.py b/tests/dopamine/replay_memory/circular_replay_buffer_test.py
index 903ff21b..6b962677 100644
--- a/tests/dopamine/replay_memory/circular_replay_buffer_test.py
+++ b/tests/dopamine/replay_memory/circular_replay_buffer_test.py
@@ -330,10 +330,13 @@ def testSampleTransitionBatch(self):
[min((x + num_adds - replay_capacity) % 4, 1) for x in indices])
batch = memory.sample_transition_batch(batch_size=len(indices),
indices=indices)
- states, action, reward, next_states, terminal, indices_batch = batch
+ (states, action, reward, next_states, next_action, next_reward, terminal,
+ indices_batch) = batch
self.assertAllEqual(states, expected_states)
self.assertAllEqual(action, np.zeros(len(indices)))
self.assertAllEqual(reward, np.zeros(len(indices)))
+ self.assertAllEqual(next_action, np.zeros(len(indices)))
+ self.assertAllEqual(next_reward, np.zeros(len(indices)))
self.assertAllEqual(next_states, expected_next_states)
self.assertAllEqual(terminal, expected_terminal)
self.assertAllEqual(indices_batch, indices)
@@ -385,11 +388,13 @@ def testSampleTransitionBatchExtra(self):
expected_extra2 = np.zeros([len(indices), 2])
batch = memory.sample_transition_batch(
batch_size=len(indices), indices=indices)
- (states, action, reward, next_states, terminal, indices_batch, extra1,
- extra2) = batch
+ (states, action, reward, next_states, next_action, next_reward, terminal,
+ indices_batch, extra1, extra2) = batch
self.assertAllEqual(states, expected_states)
self.assertAllEqual(action, np.zeros(len(indices)))
self.assertAllEqual(reward, np.zeros(len(indices)))
+ self.assertAllEqual(next_action, np.zeros(len(indices)))
+ self.assertAllEqual(next_reward, np.zeros(len(indices)))
self.assertAllEqual(next_states, expected_next_states)
self.assertAllEqual(terminal, expected_terminal)
self.assertAllEqual(indices_batch, indices)
@@ -415,7 +420,7 @@ def testSamplingWithterminalInTrajectory(self):
indices = [2, 3, 4]
batch = memory.sample_transition_batch(batch_size=len(indices),
indices=indices)
- states, action, reward, _, terminal, indices_batch = batch
+ states, action, reward, _, _, _, terminal, indices_batch = batch
expected_states = np.array([
np.full(OBSERVATION_SHAPE + (1,), i, dtype=OBS_DTYPE)
for i in indices
@@ -494,7 +499,7 @@ def testSave(self):
memory.terminal = self._test_terminal
current_iteration = 5
stale_iteration = (
- current_iteration - circular_replay_buffer.CHECKPOINT_DURATION)
+ current_iteration - circular_replay_buffer.get_checkpoint_duration())
memory.save(self._test_subdir, stale_iteration)
for attr in memory.__dict__:
if attr.startswith('_'):
@@ -528,7 +533,7 @@ def testSaveNonNDArrayAttributes(self):
current_iteration = 5
stale_iteration = (
- current_iteration - circular_replay_buffer.CHECKPOINT_DURATION)
+ current_iteration - circular_replay_buffer.get_checkpoint_duration())
memory.save(self._test_subdir, stale_iteration)
for attr in memory.__dict__:
if attr.startswith('_'):
@@ -686,7 +691,8 @@ def testConstructorWithExtraStorageTypes(self):
])
def _verify_sampled_trajectories(self, batch):
- states, action, reward, next_states, terminal, indices = batch.values()
+ (states, action, reward, next_states, next_action, next_reward, terminal,
+ indices) = batch.values()
# Because we've added BATCH_SIZE * 2 observation, using the enumerator to
# fill the respective observation, all observation will be np.full arrays
# where the values are in [0, BATCH_SIZE * 2). Because we're sampling we can
@@ -702,6 +708,8 @@ def _verify_sampled_trajectories(self, batch):
self.assertAllClose(next_states, midpoint_observation, rtol=BATCH_SIZE)
self.assertAllClose(action, np.ones(BATCH_SIZE) * 2)
self.assertAllClose(reward, np.ones(BATCH_SIZE))
+ self.assertAllClose(next_action, np.ones(BATCH_SIZE) * 2)
+ self.assertAllClose(next_reward, np.ones(BATCH_SIZE))
self.assertAllClose(terminal, np.zeros(BATCH_SIZE))
self.assertAllClose(indices, np.ones(BATCH_SIZE) * BATCH_SIZE,
rtol=BATCH_SIZE)
diff --git a/tests/dopamine/replay_memory/prioritized_replay_buffer_test.py b/tests/dopamine/replay_memory/prioritized_replay_buffer_test.py
index 759a1ed8..db6c0e59 100644
--- a/tests/dopamine/replay_memory/prioritized_replay_buffer_test.py
+++ b/tests/dopamine/replay_memory/prioritized_replay_buffer_test.py
@@ -41,8 +41,7 @@ def create_default_memory(self):
BATCH_SIZE,
max_sample_attempts=10) # For faster tests.
- def add_blank(self, memory, action=0, reward=0.0, terminal=0,
- priority=1.):
+ def add_blank(self, memory, action=0, reward=0.0, terminal=0, priority=1.0):
"""Adds a replay transition with a blank observation.
Allows setting action, reward, terminal.
@@ -57,17 +56,14 @@ def add_blank(self, memory, action=0, reward=0.0, terminal=0,
Index of the transition just added.
"""
dummy = np.zeros(SCREEN_SIZE)
- # We use _add here to bypass the additional padding done by the replay
- # memory at the beginning of an episode.
memory.add(dummy, action, reward, terminal, priority)
- index = memory.cursor() - 1
-
+ index = (memory.cursor() - 1) % REPLAY_CAPACITY
return index
def testDummyScreensAddedToNewMemory(self):
memory = self.create_default_memory()
index = self.add_blank(memory)
- for i in range(index -1):
+ for i in range(index):
self.assertEqual(memory.sum_tree.get(i), 0.0)
def testGetPriorityWithInvalidIndices(self):
@@ -95,7 +91,6 @@ def testSetAndGetPriority(self):
def testNewElementHasHighPriority(self):
memory = self.create_default_memory()
-
index = self.add_blank(memory)
self.assertEqual(
memory.get_priority(np.array([index], dtype=np.int32))[0],
@@ -109,8 +104,8 @@ def testLowPriorityElementNotFrequentlySampled(self):
for _ in range(3):
self.add_blank(memory, terminal=1)
# This test should always pass.
- for _ in range(5):
- _, _, _, _, terminals, _, _ = (
+ for _ in range(100):
+ _, _, _, _, _, _, terminals, _, _ = (
memory.sample_transition_batch(batch_size=2))
# Ensure all terminals are set to 1.
self.assertTrue((terminals == 1).all())
@@ -124,7 +119,7 @@ def testSampleIndexBatchTooManyFailedRetries(self):
self.add_blank(memory)
with self.assertRaises(
RuntimeError,
- msg='Max saple attempsts: Tried 10 times but only sampled 1 valid '
+ msg='Max sample attempts: Tried 10 times but only sampled 1 valid '
'indices. Batch size is 2'):
memory.sample_index_batch(2)
@@ -161,8 +156,8 @@ def create_default_memory(self):
batch_size=BATCH_SIZE,
max_sample_attempts=10) # For faster tests.
- def add_blank(self, replay, sess):
- replay.add(np.zeros(SCREEN_SIZE), 0, 0, 0, 1.)
+ def add_blank(self, replay):
+ replay.add(np.zeros(SCREEN_SIZE), 0, 0, 0, 1.0)
def testSetAndGetPriority(self):
replay = self.create_default_memory()
@@ -171,7 +166,7 @@ def testSetAndGetPriority(self):
with self.test_session() as sess:
indices = np.zeros(batch_size, dtype=np.int32)
for index in range(batch_size):
- self.add_blank(replay, sess)
+ self.add_blank(replay)
indices[index] = replay.memory.cursor() - 1
priorities = np.arange(batch_size)
@@ -189,10 +184,10 @@ def testSampleBatch(self):
num_data = 64
with self.test_session() as sess:
for _ in range(num_data):
- self.add_blank(replay, sess)
+ self.add_blank(replay)
probabilities = sess.run(replay.transition['sampling_probabilities'])
for prob in probabilities:
- self.assertEqual(prob, 1.)
+ self.assertEqual(prob, 1.0)
def testConstructorWithExtraStorageTypes(self):
prioritized_replay_buffer.OutOfGraphPrioritizedReplayBuffer(
diff --git a/tests/dopamine/tests/gin_config_test.py b/tests/dopamine/tests/gin_config_test.py
index d642630d..1200082a 100644
--- a/tests/dopamine/tests/gin_config_test.py
+++ b/tests/dopamine/tests/gin_config_test.py
@@ -21,8 +21,8 @@
from absl import flags
-from dopamine.atari import run_experiment
-from dopamine.atari import train
+from dopamine.discrete_domains import atari_lib
+from dopamine.discrete_domains import run_experiment
import tensorflow as tf
import gin.tf
@@ -36,124 +36,131 @@ class GinConfigTest(tf.test.TestCase):
"""
def setUp(self):
- FLAGS.base_dir = os.path.join(
+ self._base_dir = os.path.join(
'/tmp/dopamine_tests',
datetime.datetime.utcnow().strftime('run_%Y_%m_%d_%H_%M_%S'))
- self._checkpoint_dir = os.path.join(FLAGS.base_dir, 'checkpoints')
- self._logging_dir = os.path.join(FLAGS.base_dir, 'logs')
- self._videos_dir = os.path.join(FLAGS.base_dir, 'videos')
+ self._checkpoint_dir = os.path.join(self._base_dir, 'checkpoints')
+ self._logging_dir = os.path.join(self._base_dir, 'logs')
+ self._videos_dir = os.path.join(self._base_dir, 'videos')
gin.clear_config()
def testDefaultGinDqn(self):
"""Test DQNAgent configuration using the default gin config."""
tf.logging.info('####### Training the DQN agent #####')
- tf.logging.info('####### DQN base_dir: {}'.format(FLAGS.base_dir))
- FLAGS.agent_name = 'dqn'
- FLAGS.gin_files = ['dopamine/agents/dqn/configs/dqn.gin']
- FLAGS.gin_bindings = [
- 'WrappedReplayBuffer.replay_capacity = 100' # To prevent OOM.
+ tf.logging.info('####### DQN base_dir: {}'.format(self._base_dir))
+ gin_files = ['dopamine/agents/dqn/configs/dqn.gin']
+ gin_bindings = [
+ 'WrappedReplayBuffer.replay_capacity = 100', # To prevent OOM.
+ "create_agent.agent_name = 'dqn'"
]
- run_experiment.load_gin_configs(FLAGS.gin_files, FLAGS.gin_bindings)
- runner = run_experiment.Runner(FLAGS.base_dir, train.create_agent)
+ run_experiment.load_gin_configs(gin_files, gin_bindings)
+ runner = run_experiment.Runner(self._base_dir, run_experiment.create_agent,
+ atari_lib.create_atari_environment)
self.assertIsInstance(runner._agent.optimizer, tf.train.RMSPropOptimizer)
self.assertNear(0.00025, runner._agent.optimizer._learning_rate, 0.0001)
- shutil.rmtree(FLAGS.base_dir)
+ shutil.rmtree(self._base_dir)
def testOverrideRunnerParams(self):
"""Test DQNAgent configuration using the default gin config."""
tf.logging.info('####### Training the DQN agent #####')
- tf.logging.info('####### DQN base_dir: {}'.format(FLAGS.base_dir))
- FLAGS.agent_name = 'dqn'
- FLAGS.gin_files = ['dopamine/agents/dqn/configs/dqn.gin']
- FLAGS.gin_bindings = [
- 'TrainRunner.base_dir = "{}"'.format(FLAGS.base_dir),
+ tf.logging.info('####### DQN base_dir: {}'.format(self._base_dir))
+ gin_files = ['dopamine/agents/dqn/configs/dqn.gin']
+ gin_bindings = [
+ 'TrainRunner.base_dir = "{}"'.format(self._base_dir),
'Runner.log_every_n = 1729',
- 'WrappedReplayBuffer.replay_capacity = 100' # To prevent OOM.
+ 'WrappedReplayBuffer.replay_capacity = 100', # To prevent OOM.
+ "create_agent.agent_name = 'dqn'"
]
- run_experiment.load_gin_configs(FLAGS.gin_files, FLAGS.gin_bindings)
- runner = run_experiment.TrainRunner(create_agent_fn=train.create_agent)
- self.assertEqual(runner._base_dir, FLAGS.base_dir)
+ run_experiment.load_gin_configs(gin_files, gin_bindings)
+ runner = run_experiment.TrainRunner(
+ create_agent_fn=run_experiment.create_agent,
+ create_environment_fn=atari_lib.create_atari_environment)
+ self.assertEqual(runner._base_dir, self._base_dir)
self.assertEqual(runner._log_every_n, 1729)
- shutil.rmtree(FLAGS.base_dir)
+ shutil.rmtree(self._base_dir)
def testDefaultGinRmspropDqn(self):
"""Test DQNAgent configuration overridden with RMSPropOptimizer."""
tf.logging.info('####### Training the DQN agent #####')
- tf.logging.info('####### DQN base_dir: {}'.format(FLAGS.base_dir))
- FLAGS.agent_name = 'dqn'
- FLAGS.gin_files = ['dopamine/agents/dqn/configs/dqn.gin']
- FLAGS.gin_bindings = [
+ tf.logging.info('####### DQN base_dir: {}'.format(self._base_dir))
+ gin_files = ['dopamine/agents/dqn/configs/dqn.gin']
+ gin_bindings = [
'DQNAgent.optimizer = @tf.train.RMSPropOptimizer()',
'tf.train.RMSPropOptimizer.learning_rate = 100',
- 'WrappedReplayBuffer.replay_capacity = 100' # To prevent OOM.
+ 'WrappedReplayBuffer.replay_capacity = 100', # To prevent OOM.
+ "create_agent.agent_name = 'dqn'"
]
- run_experiment.load_gin_configs(FLAGS.gin_files, FLAGS.gin_bindings)
- runner = run_experiment.Runner(FLAGS.base_dir, train.create_agent)
+ run_experiment.load_gin_configs(gin_files, gin_bindings)
+ runner = run_experiment.Runner(self._base_dir, run_experiment.create_agent,
+ atari_lib.create_atari_environment)
self.assertIsInstance(runner._agent.optimizer, tf.train.RMSPropOptimizer)
self.assertEqual(100, runner._agent.optimizer._learning_rate)
- shutil.rmtree(FLAGS.base_dir)
+ shutil.rmtree(self._base_dir)
def testOverrideGinDqn(self):
"""Test DQNAgent configuration overridden with AdamOptimizer."""
tf.logging.info('####### Training the DQN agent #####')
- tf.logging.info('####### DQN base_dir: {}'.format(FLAGS.base_dir))
- FLAGS.agent_name = 'dqn'
- FLAGS.gin_files = ['dopamine/agents/dqn/configs/dqn.gin']
- FLAGS.gin_bindings = [
+ tf.logging.info('####### DQN base_dir: {}'.format(self._base_dir))
+ gin_files = ['dopamine/agents/dqn/configs/dqn.gin']
+ gin_bindings = [
'DQNAgent.optimizer = @tf.train.AdamOptimizer()',
'tf.train.AdamOptimizer.learning_rate = 100',
- 'WrappedReplayBuffer.replay_capacity = 100' # To prevent OOM.
+ 'WrappedReplayBuffer.replay_capacity = 100', # To prevent OOM.
+ "create_agent.agent_name = 'dqn'"
]
- run_experiment.load_gin_configs(FLAGS.gin_files, FLAGS.gin_bindings)
- runner = run_experiment.Runner(FLAGS.base_dir, train.create_agent)
+ run_experiment.load_gin_configs(gin_files, gin_bindings)
+ runner = run_experiment.Runner(self._base_dir, run_experiment.create_agent,
+ atari_lib.create_atari_environment)
self.assertIsInstance(runner._agent.optimizer, tf.train.AdamOptimizer)
self.assertEqual(100, runner._agent.optimizer._lr)
- shutil.rmtree(FLAGS.base_dir)
+ shutil.rmtree(self._base_dir)
def testDefaultGinRainbow(self):
"""Test RainbowAgent default configuration using default gin."""
tf.logging.info('####### Training the RAINBOW agent #####')
- tf.logging.info('####### RAINBOW base_dir: {}'.format(FLAGS.base_dir))
- FLAGS.agent_name = 'rainbow'
- FLAGS.gin_files = [
+ tf.logging.info('####### RAINBOW base_dir: {}'.format(self._base_dir))
+ gin_files = [
'dopamine/agents/rainbow/configs/rainbow.gin'
]
- FLAGS.gin_bindings = [
- 'WrappedReplayBuffer.replay_capacity = 100' # To prevent OOM.
+ gin_bindings = [
+ 'WrappedReplayBuffer.replay_capacity = 100', # To prevent OOM.
+ "create_agent.agent_name = 'rainbow'"
]
- run_experiment.load_gin_configs(FLAGS.gin_files, FLAGS.gin_bindings)
- runner = run_experiment.Runner(FLAGS.base_dir, train.create_agent)
+ run_experiment.load_gin_configs(gin_files, gin_bindings)
+ runner = run_experiment.Runner(self._base_dir, run_experiment.create_agent,
+ atari_lib.create_atari_environment)
self.assertIsInstance(runner._agent.optimizer, tf.train.AdamOptimizer)
self.assertNear(0.0000625, runner._agent.optimizer._lr, 0.0001)
- shutil.rmtree(FLAGS.base_dir)
+ shutil.rmtree(self._base_dir)
def testOverrideGinRainbow(self):
"""Test RainbowAgent configuration overridden with RMSPropOptimizer."""
tf.logging.info('####### Training the RAINBOW agent #####')
- tf.logging.info('####### RAINBOW base_dir: {}'.format(FLAGS.base_dir))
- FLAGS.agent_name = 'rainbow'
- FLAGS.gin_files = [
+ tf.logging.info('####### RAINBOW base_dir: {}'.format(self._base_dir))
+ gin_files = [
'dopamine/agents/rainbow/configs/rainbow.gin',
]
- FLAGS.gin_bindings = [
+ gin_bindings = [
'RainbowAgent.optimizer = @tf.train.RMSPropOptimizer()',
'tf.train.RMSPropOptimizer.learning_rate = 100',
- 'WrappedReplayBuffer.replay_capacity = 100' # To prevent OOM.
+ 'WrappedReplayBuffer.replay_capacity = 100', # To prevent OOM.
+ "create_agent.agent_name = 'rainbow'"
]
- run_experiment.load_gin_configs(FLAGS.gin_files, FLAGS.gin_bindings)
- runner = run_experiment.Runner(FLAGS.base_dir, train.create_agent)
+ run_experiment.load_gin_configs(gin_files, gin_bindings)
+ runner = run_experiment.Runner(self._base_dir, run_experiment.create_agent,
+ atari_lib.create_atari_environment)
self.assertIsInstance(runner._agent.optimizer, tf.train.RMSPropOptimizer)
self.assertEqual(100, runner._agent.optimizer._learning_rate)
- shutil.rmtree(FLAGS.base_dir)
+ shutil.rmtree(self._base_dir)
def testDefaultDQNConfig(self):
"""Verify the default DQN configuration."""
- FLAGS.agent_name = 'dqn'
run_experiment.load_gin_configs(
['dopamine/agents/dqn/configs/dqn.gin'], [])
- agent = train.create_agent(self.test_session(),
- run_experiment.create_atari_environment('Pong'))
+ agent = run_experiment.create_agent(
+ self.test_session(),
+ atari_lib.create_atari_environment(game_name='Pong'))
self.assertEqual(agent.gamma, 0.99)
self.assertEqual(agent.update_horizon, 1)
self.assertEqual(agent.min_replay_history, 20000)
@@ -167,11 +174,11 @@ def testDefaultDQNConfig(self):
def testDefaultC51Config(self):
"""Verify the default C51 configuration."""
- FLAGS.agent_name = 'rainbow'
run_experiment.load_gin_configs(
['dopamine/agents/rainbow/configs/c51.gin'], [])
- agent = train.create_agent(self.test_session(),
- run_experiment.create_atari_environment('Pong'))
+ agent = run_experiment.create_agent(
+ self.test_session(),
+ atari_lib.create_atari_environment(game_name='Pong'))
self.assertEqual(agent._num_atoms, 51)
support = self.evaluate(agent._support)
self.assertEqual(min(support), -10.)
@@ -190,11 +197,11 @@ def testDefaultC51Config(self):
def testDefaultRainbowConfig(self):
"""Verify the default Rainbow configuration."""
- FLAGS.agent_name = 'rainbow'
run_experiment.load_gin_configs(
['dopamine/agents/rainbow/configs/rainbow.gin'], [])
- agent = train.create_agent(self.test_session(),
- run_experiment.create_atari_environment('Pong'))
+ agent = run_experiment.create_agent(
+ self.test_session(),
+ atari_lib.create_atari_environment(game_name='Pong'))
self.assertEqual(agent._num_atoms, 51)
support = self.evaluate(agent._support)
self.assertEqual(min(support), -10.)
@@ -214,58 +221,58 @@ def testDefaultRainbowConfig(self):
def testDefaultGinImplicitQuantileIcml(self):
"""Test default ImplicitQuantile configuration using ICML gin."""
tf.logging.info('###### Training the Implicit Quantile agent #####')
- FLAGS.agent_name = 'implicit_quantile'
- FLAGS.base_dir = os.path.join(
+ self._base_dir = os.path.join(
'/tmp/dopamine_tests',
datetime.datetime.utcnow().strftime('run_%Y_%m_%d_%H_%M_%S'))
- tf.logging.info('###### IQN base dir: {}'.format(FLAGS.base_dir))
- FLAGS.gin_files = ['dopamine/agents/'
- 'implicit_quantile/configs/implicit_quantile_icml.gin']
- FLAGS.gin_bindings = [
+ tf.logging.info('###### IQN base dir: {}'.format(self._base_dir))
+ gin_files = ['dopamine/agents/'
+ 'implicit_quantile/configs/implicit_quantile_icml.gin']
+ gin_bindings = [
'Runner.num_iterations=0',
]
- run_experiment.load_gin_configs(FLAGS.gin_files, FLAGS.gin_bindings)
- runner = run_experiment.Runner(FLAGS.base_dir, train.create_agent)
+ run_experiment.load_gin_configs(gin_files, gin_bindings)
+ runner = run_experiment.Runner(self._base_dir, run_experiment.create_agent,
+ atari_lib.create_atari_environment)
self.assertEqual(1000000, runner._agent._replay.memory._replay_capacity)
- shutil.rmtree(FLAGS.base_dir)
+ shutil.rmtree(self._base_dir)
def testOverrideGinImplicitQuantileIcml(self):
"""Test ImplicitQuantile configuration overriding using ICML gin."""
tf.logging.info('###### Training the Implicit Quantile agent #####')
- FLAGS.agent_name = 'implicit_quantile'
- FLAGS.base_dir = os.path.join(
+ self._base_dir = os.path.join(
'/tmp/dopamine_tests',
datetime.datetime.utcnow().strftime('run_%Y_%m_%d_%H_%M_%S'))
- tf.logging.info('###### IQN base dir: {}'.format(FLAGS.base_dir))
- FLAGS.gin_files = ['dopamine/agents/'
- 'implicit_quantile/configs/implicit_quantile_icml.gin']
- FLAGS.gin_bindings = [
+ tf.logging.info('###### IQN base dir: {}'.format(self._base_dir))
+ gin_files = ['dopamine/agents/implicit_quantile/configs/'
+ 'implicit_quantile_icml.gin']
+ gin_bindings = [
'Runner.num_iterations=0',
'WrappedPrioritizedReplayBuffer.replay_capacity = 1000',
]
- run_experiment.load_gin_configs(FLAGS.gin_files, FLAGS.gin_bindings)
- runner = run_experiment.Runner(FLAGS.base_dir, train.create_agent)
+ run_experiment.load_gin_configs(gin_files, gin_bindings)
+ runner = run_experiment.Runner(self._base_dir, run_experiment.create_agent,
+ atari_lib.create_atari_environment)
self.assertEqual(1000, runner._agent._replay.memory._replay_capacity)
- shutil.rmtree(FLAGS.base_dir)
+ shutil.rmtree(self._base_dir)
def testOverrideGinImplicitQuantile(self):
"""Test ImplicitQuantile configuration overriding using IQN gin."""
tf.logging.info('###### Training the Implicit Quantile agent #####')
- FLAGS.agent_name = 'implicit_quantile'
- FLAGS.base_dir = os.path.join(
+ self._base_dir = os.path.join(
'/tmp/dopamine_tests',
datetime.datetime.utcnow().strftime('run_%Y_%m_%d_%H_%M_%S'))
- tf.logging.info('###### IQN base dir: {}'.format(FLAGS.base_dir))
- FLAGS.gin_files = ['dopamine/agents/'
- 'implicit_quantile/configs/implicit_quantile.gin']
- FLAGS.gin_bindings = [
+ tf.logging.info('###### IQN base dir: {}'.format(self._base_dir))
+ gin_files = ['dopamine/agents/implicit_quantile/configs/'
+ 'implicit_quantile.gin']
+ gin_bindings = [
'Runner.num_iterations=0',
'WrappedPrioritizedReplayBuffer.replay_capacity = 1000',
]
- run_experiment.load_gin_configs(FLAGS.gin_files, FLAGS.gin_bindings)
- runner = run_experiment.Runner(FLAGS.base_dir, train.create_agent)
+ run_experiment.load_gin_configs(gin_files, gin_bindings)
+ runner = run_experiment.Runner(self._base_dir, run_experiment.create_agent,
+ atari_lib.create_atari_environment)
self.assertEqual(1000, runner._agent._replay.memory._replay_capacity)
- shutil.rmtree(FLAGS.base_dir)
+ shutil.rmtree(self._base_dir)
if __name__ == '__main__':
diff --git a/tests/dopamine/tests/integration_test.py b/tests/dopamine/tests/integration_test.py
index 3cb36047..04661209 100644
--- a/tests/dopamine/tests/integration_test.py
+++ b/tests/dopamine/tests/integration_test.py
@@ -21,7 +21,7 @@
from absl import flags
-from dopamine.atari import train
+from dopamine.discrete_domains import train
import tensorflow as tf
import gin.tf
@@ -46,24 +46,26 @@ def setUp(self):
def quickDqnFlags(self):
"""Assign flags for a quick run of DQNAgent."""
- FLAGS.agent_name = 'dqn'
FLAGS.gin_files = ['dopamine/agents/dqn/configs/dqn.gin']
FLAGS.gin_bindings = [
- 'Runner.training_steps=100', 'Runner.evaluation_steps=10',
- 'Runner.num_iterations=1', 'Runner.max_steps_per_episode=100',
+ 'Runner.training_steps=100',
+ 'Runner.evaluation_steps=10',
+ 'Runner.num_iterations=1',
+ 'Runner.max_steps_per_episode=100',
'dqn_agent.DQNAgent.min_replay_history=500',
'WrappedReplayBuffer.replay_capacity=100'
]
def quickRainbowFlags(self):
"""Assign flags for a quick run of RainbowAgent."""
- FLAGS.agent_name = 'rainbow'
FLAGS.gin_files = [
'dopamine/agents/rainbow/configs/rainbow.gin'
]
FLAGS.gin_bindings = [
- 'Runner.training_steps=100', 'Runner.evaluation_steps=10',
- 'Runner.num_iterations=1', 'Runner.max_steps_per_episode=100',
+ 'Runner.training_steps=100',
+ 'Runner.evaluation_steps=10',
+ 'Runner.num_iterations=1',
+ 'Runner.max_steps_per_episode=100',
"rainbow_agent.RainbowAgent.replay_scheme='prioritized'",
'rainbow_agent.RainbowAgent.min_replay_history=500',
'WrappedReplayBuffer.replay_capacity=100'
diff --git a/tests/dopamine/tests/train_runner_integration_test.py b/tests/dopamine/tests/train_runner_integration_test.py
index 080c750b..f259bce4 100644
--- a/tests/dopamine/tests/train_runner_integration_test.py
+++ b/tests/dopamine/tests/train_runner_integration_test.py
@@ -22,7 +22,7 @@
from absl import flags
-from dopamine.atari import train
+from dopamine.discrete_domains import train
import tensorflow as tf
FLAGS = flags.FLAGS
@@ -39,15 +39,16 @@ def setUp(self):
datetime.datetime.utcnow().strftime('run_%Y_%m_%d_%H_%M_%S'))
self._checkpoint_dir = os.path.join(FLAGS.base_dir, 'checkpoints')
self._logging_dir = os.path.join(FLAGS.base_dir, 'logs')
- FLAGS.schedule = 'continuous_train'
def quickDqnFlags(self):
"""Assign flags for a quick run of DQN agent."""
- FLAGS.agent_name = 'dqn'
FLAGS.gin_files = ['dopamine/agents/dqn/configs/dqn.gin']
FLAGS.gin_bindings = [
- 'Runner.training_steps=100', 'Runner.evaluation_steps=10',
- 'Runner.num_iterations=1', 'Runner.max_steps_per_episode=100',
+ "create_runner.schedule='continuous_train'",
+ 'Runner.training_steps=100',
+ 'Runner.evaluation_steps=10',
+ 'Runner.num_iterations=1',
+ 'Runner.max_steps_per_episode=100',
'dqn_agent.DQNAgent.min_replay_history=500',
'WrappedReplayBuffer.replay_capacity=100'
]