From 2ef1999d3df25e9e1c8648f9a697eca8341a0abd Mon Sep 17 00:00:00 2001 From: Cristian Garcia Date: Tue, 14 Jan 2025 14:51:24 +0000 Subject: [PATCH] [nnx] add cache_args --- .github/workflows/flax_publish.yml | 4 +- .github/workflows/flax_test.yml | 42 +- .github/workflows/flaxlib_publish.yml | 18 +- .github/workflows/jax_nightly.yml | 52 - benchmarks/nnx_graph_overhead.py | 63 +- benchmarks/nnx_mlpmixer_training.py | 235 +++ benchmarks/nnx_simple_training.py | 81 +- docs_nnx/guides/checkpointing.ipynb | 22 +- docs_nnx/guides/filters_guide.ipynb | 94 +- docs_nnx/guides/filters_guide.md | 94 +- docs_nnx/mnist_tutorial.ipynb | 235 ++- docs_nnx/mnist_tutorial.md | 49 +- docs_nnx/nnx_basics.ipynb | 122 +- docs_nnx/nnx_basics.md | 6 +- .../nnx_toy_examples/02_lifted_transforms.py | 6 +- flax/configurations.py | 11 + flax/linen/module.py | 5 - flax/nnx/__init__.py | 1 + flax/nnx/bridge/variables.py | 21 +- flax/nnx/extract.py | 111 +- flax/nnx/filterlib.py | 4 +- flax/nnx/graph.py | 1457 +++++++++++++---- flax/nnx/helpers.py | 4 + flax/nnx/module.py | 17 + flax/nnx/nn/linear.py | 2 +- flax/nnx/nn/normalization.py | 10 +- flax/nnx/nn/recurrent.py | 80 +- flax/nnx/nn/stochastic.py | 5 +- flax/nnx/object.py | 165 +- flax/nnx/reprlib.py | 203 +-- flax/nnx/rnglib.py | 34 +- flax/nnx/statelib.py | 132 +- flax/nnx/tracers.py | 13 +- flax/nnx/training/metrics.py | 26 +- flax/nnx/transforms/autodiff.py | 122 +- flax/nnx/transforms/compilation.py | 64 +- flax/nnx/transforms/general.py | 6 +- flax/nnx/transforms/iteration.py | 147 +- flax/nnx/transforms/transforms.py | 3 +- flax/nnx/variablelib.py | 116 +- flax/nnx/visualization.py | 112 +- flax/struct.py | 2 +- flax/typing.py | 63 - flaxlib_src/CMakeLists.txt | 54 + flaxlib_src/meson.build | 14 - flaxlib_src/pyproject.toml | 17 +- .../{flaxlib.pyi => src/flaxlib/__init__.py} | 3 +- flaxlib_src/src/flaxlib/flaxlib_cpp.pyi | 25 + flaxlib_src/src/lib.cc | 300 +++- flaxlib_src/src/lib.rs | 28 - pyproject.toml | 8 +- tests/jax_utils_test.py | 16 +- tests/nnx/bridge/wrappers_test.py | 4 +- tests/nnx/graph_utils_test.py | 195 ++- tests/nnx/module_test.py | 60 +- tests/nnx/nn/recurrent_test.py | 1101 ++++++------- tests/nnx/transforms_test.py | 70 +- uv.lock | 53 +- 58 files changed, 3548 insertions(+), 2459 deletions(-) delete mode 100644 .github/workflows/jax_nightly.yml create mode 100644 benchmarks/nnx_mlpmixer_training.py create mode 100644 flaxlib_src/CMakeLists.txt delete mode 100644 flaxlib_src/meson.build rename flaxlib_src/{flaxlib.pyi => src/flaxlib/__init__.py} (84%) create mode 100644 flaxlib_src/src/flaxlib/flaxlib_cpp.pyi delete mode 100644 flaxlib_src/src/lib.rs diff --git a/.github/workflows/flax_publish.yml b/.github/workflows/flax_publish.yml index f688e8fc81..383461a5e7 100644 --- a/.github/workflows/flax_publish.yml +++ b/.github/workflows/flax_publish.yml @@ -13,9 +13,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@v2 - name: Set up Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + uses: actions/setup-python@v1 with: python-version: '3.x' - name: Install dependencies diff --git a/.github/workflows/flax_test.yml b/.github/workflows/flax_test.yml index 4c7993d455..4bed8d8179 100644 --- a/.github/workflows/flax_test.yml +++ b/.github/workflows/flax_test.yml @@ -3,10 +3,6 @@ name: Flax - Test -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - on: push: branches: @@ -17,26 +13,30 @@ on: - main jobs: + cancel-previous: + name: Cancel Previous Runs + runs-on: ubuntu-latest + steps: + - name: Cancel previous + uses: styfle/cancel-workflow-action@0.10.1 + if: ${{github.ref != 'refs/head/main'}} + with: + access_token: ${{ github.token }} pre-commit: name: Test pre-commit hooks runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + uses: actions/setup-python@v4 with: python-version: '3.10' - - run: python -m pip install pre-commit - - uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 - with: - path: ~/.cache/pre-commit - key: pre-commit-${{ env.pythonLocation }}-${{ hashFiles('.pre-commit-config.yaml', 'pyproject.toml') }} - - run: pre-commit run --show-diff-on-failure --color=always --all-files + - uses: pre-commit/action@v2.0.3 commit-count: name: Check commit count runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@v3 # We allow at most 5 commits in a branch to ensure our CI doesn't break. - name: Check commit count in PR if: always() @@ -65,12 +65,12 @@ jobs: matrix: python-version: ['3.10', '3.11'] steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - uses: astral-sh/setup-uv@887a942a15af3a7626099df99e897a18d9e5ab3a # v5.1.0 + - uses: astral-sh/setup-uv@v2 with: uv-version: "0.3.0" - name: Install standalone dependencies only @@ -81,7 +81,7 @@ jobs: uv run python -c "import flax" tests: name: Run Tests - needs: [pre-commit, commit-count, test-import] + needs: [cancel-previous, pre-commit, commit-count, test-import] runs-on: ubuntu-20.04-16core strategy: matrix: @@ -98,14 +98,14 @@ jobs: test-type: pytest jax-version: '0.4.27' # keep in sync with jax pin in pyproject.toml steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} id: setup_python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Setup uv - uses: astral-sh/setup-uv@887a942a15af3a7626099df99e897a18d9e5ab3a # v5.1.0 + uses: astral-sh/setup-uv@v2 with: version: "0.3.0" @@ -135,7 +135,7 @@ jobs: fi - name: Upload coverage to Codecov if: matrix.test-type == 'pytest' - uses: codecov/codecov-action@1e68e06f1dbfde0e4cefc87efeba9e4643565303 # v5.1.2 + uses: codecov/codecov-action@v4 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: diff --git a/.github/workflows/flaxlib_publish.yml b/.github/workflows/flaxlib_publish.yml index dcd017adfb..480f25902a 100644 --- a/.github/workflows/flaxlib_publish.yml +++ b/.github/workflows/flaxlib_publish.yml @@ -19,12 +19,12 @@ jobs: os: [ubuntu-latest, windows-latest, macos-13, macos-14] steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@v4 - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + - uses: actions/setup-python@v5 - name: Setup Rust - uses: actions-rust-lang/setup-rust-toolchain@11df97af8e8102fd60b60a77dfbf58d40cd843b8 # v1.10.1 + uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Install cibuildwheel run: python -m pip install cibuildwheel==2.21.0 @@ -41,7 +41,7 @@ jobs: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain=stable --profile=minimal -y && rustup show - - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + - uses: actions/upload-artifact@v4 with: name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} path: ./flaxlib/wheelhouse/*.whl @@ -51,15 +51,15 @@ jobs: name: Build source distribution runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@v4 - name: Setup Rust - uses: actions-rust-lang/setup-rust-toolchain@11df97af8e8102fd60b60a77dfbf58d40cd843b8 # v1.10.1 + uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Build sdist run: pipx run build --sdist flaxlib - - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + - uses: actions/upload-artifact@v4 with: name: cibw-sdist path: ./flaxlib/dist/*.tar.gz @@ -72,14 +72,14 @@ jobs: permissions: id-token: write steps: - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + - uses: actions/setup-python@v1 with: python-version: '3.x' - name: Install dependencies run: | python -m pip install --upgrade pip pip install setuptools build wheel twine - - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + - uses: actions/download-artifact@v4 with: # unpacks all CIBW artifacts into dist/ pattern: cibw-* diff --git a/.github/workflows/jax_nightly.yml b/.github/workflows/jax_nightly.yml deleted file mode 100644 index 7beb35e381..0000000000 --- a/.github/workflows/jax_nightly.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: CI - with JAX nightly - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - schedule: - - cron: "0 12 * * *" # Daily at 12:00 UTC - workflow_dispatch: # allows triggering the workflow run manually - pull_request: # Automatically trigger on pull requests affecting this file - branches: - - main - paths: - - '**workflows/jax_nightly.yml' - -jobs: - jax-nightly: - runs-on: ubuntu-latest - permissions: - contents: read - issues: write # for failed-build-issue - strategy: - fail-fast: false - matrix: - python-version: ["3.11"] - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Set up Python ${{ matrix.python-version }} - id: setup_python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 - with: - python-version: ${{ matrix.python-version }} - - name: Setup uv - uses: astral-sh/setup-uv@887a942a15af3a7626099df99e897a18d9e5ab3a # v5.1.0 - with: - version: "0.3.0" - - name: Install dependencies - run: | - uv sync --extra all --extra testing --extra docs - - name: Install JAX - run: | - uv pip install -U --pre jax jaxlib -f https://storage.googleapis.com/jax-releases/jax_nightly_releases.html - - name: Run test suite - if: success() - run: | - uv run tests/run_all_tests.sh --only-pytest - - name: Notify failed build - uses: jayqi/failed-build-issue-action@1a893bbf43ef1c2a8705e2b115cd4f0fe3c5649b # v1.2.0 - if: failure() && github.event.pull_request == null - with: - github-token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/benchmarks/nnx_graph_overhead.py b/benchmarks/nnx_graph_overhead.py index 88809f7775..6d10f79e07 100644 --- a/benchmarks/nnx_graph_overhead.py +++ b/benchmarks/nnx_graph_overhead.py @@ -24,31 +24,52 @@ from absl import app FLAGS = flags.FLAGS -flags.DEFINE_enum('mode', 'all', ['all', 'nnx', 'jax'], 'Mode to run the script in') +flags.DEFINE_enum( + 'mode', 'nnx', ['all', 'nnx', 'jax'], 'Mode to run the script in' +) flags.DEFINE_integer('total_steps', 100, 'Total number of training steps') flags.DEFINE_integer('width', 32, 'Hidden layer size') flags.DEFINE_integer('depth', 5, 'Depth of the model') - class Linear(nnx.Module): def __init__(self, din: int, dout: int, *, rngs: nnx.Rngs): - self.list = [ - nnx.Param(jax.random.uniform(rngs.params(), (din, dout))), - nnx.Param(jnp.zeros((dout,))), - ] - self.dict = { - 'w': nnx.Param(jax.random.uniform(rngs.params(), (din, dout))), - 'b': nnx.Param(jnp.zeros((dout,))), - } + self.w = nnx.Param(jax.random.uniform(rngs.params(), (din, dout))) + self.b = nnx.Param(jnp.zeros((dout,))) + + def __call__(self, x): + return x @ self.w + self.b + + +class Block(nnx.Module): + def __init__(self, din: int, dout: int, *, rngs: nnx.Rngs): + self.linear = Linear(din, dout, rngs=rngs) + self.bn = nnx.BatchNorm(dout, rngs=rngs) + + def __call__(self, x): + return nnx.relu(self.bn(self.linear(x))) + +class Count(nnx.Variable): + pass class MLP(nnx.Module): - def __init__(self, depth, *, rngs: nnx.Rngs): + def __init__(self, din, dhidden, dout, depth, *, rngs: nnx.Rngs): + self.count = Count(jnp.array(0)) + self.linear_in = Block(din, dhidden, rngs=rngs) self.intermediates = [ - Linear(10, 10, rngs=rngs) for _ in range(depth) + Block(dhidden, dhidden, rngs=rngs) for _ in range(depth - 2) ] + self.linear_out = Block(dhidden, dout, rngs=rngs) + + def __call__(self, x): + self.count.value += 1 + x = nnx.relu(self.linear_in(x)) + for layer in self.intermediates: + x = nnx.relu(layer(x)) + x = self.linear_out(x) + return x def main(argv): @@ -63,21 +84,24 @@ def main(argv): X = np.linspace(0, 1, 100)[:, None] Y = 0.8 * X**2 + 0.1 + np.random.normal(0, 0.1, size=X.shape) - model = MLP(depth=depth, rngs=nnx.Rngs(0)) - tx = optax.sgd(1e-3) - optimizer = nnx.Optimizer(model, tx) - #------------------------------------------------------------ # NNX #------------------------------------------------------------ if mode in ['all', 'nnx']: + model = MLP(din=1, dhidden=width, dout=1, depth=depth, rngs=nnx.Rngs(0)) + tx = optax.sgd(1e-3) + optimizer = nnx.Optimizer(model, tx) + t0 = time() + @nnx.jit def step_nnx(model: MLP, optimizer: nnx.Optimizer): pass + cached_step_nnx = nnx.cache_args(step_nnx, model, optimizer) + t0 = time() for _ in range(total_steps): - step_nnx(model, optimizer) + cached_step_nnx() total_time = time() - t0 time_per_step = total_time / total_steps @@ -93,6 +117,11 @@ def step_nnx(model: MLP, optimizer: nnx.Optimizer): #------------------------------------------------------------ if mode in ['all', 'jax']: + model = MLP(din=1, dhidden=width, dout=1, depth=depth, rngs=nnx.Rngs(0)) + tx = optax.sgd(1e-3) + optimizer = nnx.Optimizer(model, tx) + t0 = time() + @jax.jit def step_jax(graphdef, state): return graphdef, state diff --git a/benchmarks/nnx_mlpmixer_training.py b/benchmarks/nnx_mlpmixer_training.py new file mode 100644 index 0000000000..68d5e79734 --- /dev/null +++ b/benchmarks/nnx_mlpmixer_training.py @@ -0,0 +1,235 @@ +# Copyright 2024 The Flax Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# %% +from functools import partial +import jax +import jax.numpy as jnp +from flax import nnx +import optax +import numpy as np +from einop import einop +from time import time +from tqdm import tqdm + +from flax import nnx + +from absl import flags +from absl import app + +FLAGS = flags.FLAGS +flags.DEFINE_enum( + 'mode', 'all', ['all', 'nnx', 'jax'], 'Mode to run the script in' +) +flags.DEFINE_integer('total_steps', 10_000, 'Total number of training steps') +flags.DEFINE_integer('batch_size', 32, 'Batch size') +flags.DEFINE_integer('width', 32, 'Hidden layer size') +flags.DEFINE_integer('depth', 4, 'Depth of the model') + + +class MlpBlock(nnx.Module): + def __init__(self, din: int, mlp_dim: int, rngs: nnx.Rngs): + self.din, self.mlp_dim = din, mlp_dim + self.linear_in = nnx.Linear(din, mlp_dim, rngs=rngs) + self.linear_out = nnx.Linear(mlp_dim, din, rngs=rngs) + + def __call__(self, x): + return self.linear_out(nnx.gelu(self.linear_in(x))) + + +class MixerBlock(nnx.Module): + def __init__( + self, + tokens_mlp_dim: int, + channels_mlp_dim: int, + hidden_dim: int, + rngs: nnx.Rngs, + ): + self.tokens_mlp_dim = tokens_mlp_dim + self.channels_mlp_dim = channels_mlp_dim + self.hidden_dim = hidden_dim + self.token_mixing = MlpBlock(tokens_mlp_dim, hidden_dim, rngs=rngs) + self.channel_mixing = MlpBlock(channels_mlp_dim, hidden_dim, rngs=rngs) + self.ln1 = nnx.LayerNorm(channels_mlp_dim, rngs=rngs) + self.ln2 = nnx.LayerNorm(channels_mlp_dim, rngs=rngs) + + def __call__(self, x): + y = self.ln1(x) + y = y.swapaxes(1, 2) + y = self.token_mixing(y) + y = y.swapaxes(1, 2) + x = x + y + y = self.ln2(x) + return x + self.channel_mixing(y) + + +class MlpMixer(nnx.Module): + def __init__( + self, + din: int, + kernel_size: tuple[int, int], + strides: tuple[int, int], + num_blocks: int, + hidden_dim: int, + tokens_mlp_dim: int, + channels_mlp_dim: int, + rngs: nnx.Rngs, + ): + self.din = din + self.kernel_size = kernel_size + self.num_blocks = num_blocks + self.hidden_dim = hidden_dim + self.tokens_mlp_dim = tokens_mlp_dim + self.channels_mlp_dim = channels_mlp_dim + self.stem = nnx.Conv( + din + 1, + channels_mlp_dim, + kernel_size=kernel_size, + strides=strides, + rngs=rngs, + ) + self.blocks = [ + MixerBlock(tokens_mlp_dim, channels_mlp_dim, hidden_dim, rngs=rngs) + for _ in range(num_blocks) + ] + self.pre_head_layer_norm = nnx.LayerNorm(channels_mlp_dim, rngs=rngs) + self.conv_t = nnx.ConvTranspose( + channels_mlp_dim, din, kernel_size=kernel_size, strides=strides, rngs=rngs + ) + + def __call__(self, *, x, t): + # add time feature to input + t = einop(t, 'n -> n h w c', h=x.shape[1], w=x.shape[2], c=1) + x = jnp.concatenate([x, t], axis=-1) + # create patches + x = self.stem(x) + h, w = x.shape[1], x.shape[2] + x = einop(x, 'n h w c -> n (h w) c') + # apply blocks + for block in self.blocks: + x = block(x) + x = self.pre_head_layer_norm(x) + # recreate image + x = einop(x, 'n (h w) c -> n h w c', h=h, w=w) + x = self.conv_t(x) + return x + + +def main(argv): + print(argv) + mode: str = FLAGS.mode + total_steps: int = FLAGS.total_steps + batch_size: int = FLAGS.batch_size + width: int = FLAGS.width + depth: int = FLAGS.depth + + print(f'{mode=}, {total_steps=}, {batch_size=}, {width=}') + + X = np.random.uniform(size=(batch_size, 28, 28, 1)) + + if mode == 'nnx' or mode == 'all': + rngs = nnx.Rngs(0) + flow = MlpMixer( + din=1, + kernel_size=(2, 2), + strides=(2, 2), + num_blocks=4, + hidden_dim=512, + tokens_mlp_dim=196, + channels_mlp_dim=512, + rngs=rngs, + ) + optimizer = nnx.Optimizer(flow, tx=optax.adamw(1e-4)) + t0 = time() + + mse = lambda a, b: jnp.mean((a - b) ** 2) + + @nnx.jit(donate_argnums=(0, 1, 2)) + def train_step_nnx(flow, optimizer, rngs, x_1): + print('JITTING NNX') + x_0 = jax.random.normal(rngs(), x_1.shape) + t = jax.random.uniform(rngs(), (len(x_1),)) + + x_t = jax.vmap(lambda x_0, x_1, t: (1 - t) * x_0 + t * x_1)(x_0, x_1, t) + dx_t = x_1 - x_0 + + loss, grads = nnx.value_and_grad( + lambda flow: mse(flow(x=x_t, t=t), dx_t) + )(flow) + optimizer.update(grads) + return loss + + losses = [] + t0 = time() + for step in tqdm(range(total_steps), desc='NNX'): + loss = train_step_nnx(flow, optimizer, rngs, X) + losses.append(loss) + + total_time = time() - t0 + print('### NNX ###') + print(f'final loss: {losses[-1]}') + print('total time:', total_time) + print(f'time per step: {total_time / total_steps * 1e6:.2f} µs') + + if mode == 'jax' or mode == 'all': + rngs = nnx.Rngs(0) + flow = MlpMixer( + din=1, + kernel_size=(2, 2), + strides=(2, 2), + num_blocks=depth, + hidden_dim=width, + tokens_mlp_dim=196, + channels_mlp_dim=width, + rngs=rngs, + ) + optimizer = nnx.Optimizer(flow, tx=optax.adamw(1e-4)) + graphdef, state = nnx.split((flow, optimizer, rngs)) + t0 = time() + + mse = lambda a, b: jnp.mean((a - b) ** 2) + + @partial(nnx.jit, donate_argnums=0) + def train_step_jax(state, x_1): + print('JITTING JAX') + flow, optimizer, rngs = nnx.merge(graphdef, state) + x_0 = jax.random.normal(rngs(), x_1.shape) + t = jax.random.uniform(rngs(), (len(x_1),)) + + x_t = jax.vmap(lambda x_0, x_1, t: (1 - t) * x_0 + t * x_1)(x_0, x_1, t) + dx_t = x_1 - x_0 + + loss, grads = nnx.value_and_grad( + lambda flow: mse(flow(x=x_t, t=t), dx_t) + )(flow) + optimizer.update(grads) + state = nnx.state((flow, optimizer, rngs)) + return loss, state + + losses = [] + t0 = time() + for step in tqdm(range(total_steps), desc='JAX'): + loss, state = train_step_jax(state, X) + losses.append(loss) + + nnx.update((flow, optimizer, rngs), state) + total_time = time() - t0 + print('### JAX ###') + print(f'final loss: {losses[-1]}') + print('total time:', total_time) + print(f'time per step: {total_time / total_steps * 1e6:.2f} µs') + + +if __name__ == '__main__': + app.run(main) diff --git a/benchmarks/nnx_simple_training.py b/benchmarks/nnx_simple_training.py index 0cb08066fe..88195b3ffd 100644 --- a/benchmarks/nnx_simple_training.py +++ b/benchmarks/nnx_simple_training.py @@ -13,6 +13,7 @@ # limitations under the License. # %% +from functools import partial import jax import jax.numpy as jnp import numpy as np @@ -25,7 +26,9 @@ from absl import app FLAGS = flags.FLAGS -flags.DEFINE_enum('mode', 'nnx', ['nnx', 'jax'], 'Mode to run the script in') +flags.DEFINE_enum( + 'mode', 'all', ['all', 'nnx', 'jax'], 'Mode to run the script in' +) flags.DEFINE_integer('total_steps', 10_000, 'Total number of training steps') flags.DEFINE_integer('batch_size', 32, 'Batch size') flags.DEFINE_integer('width', 32, 'Hidden layer size') @@ -46,6 +49,13 @@ def __init__(self, din: int, dout: int, *, rngs: nnx.Rngs): def __call__(self, x): return x @ self.w + self.b +class Block(nnx.Module): + def __init__(self, din: int, dout: int, *, rngs: nnx.Rngs): + self.linear = Linear(din, dout, rngs=rngs) + self.bn = nnx.BatchNorm(dout, rngs=rngs) + + def __call__(self, x): + return nnx.relu(self.bn(self.linear(x))) class Count(nnx.Variable): pass @@ -54,11 +64,11 @@ class Count(nnx.Variable): class MLP(nnx.Module): def __init__(self, din, dhidden, dout, depth, *, rngs: nnx.Rngs): self.count = Count(jnp.array(0)) - self.linear_in = Linear(din, dhidden, rngs=rngs) + self.linear_in = Block(din, dhidden, rngs=rngs) self.intermediates = [ - Linear(dhidden, dhidden, rngs=rngs) for _ in range(depth - 2) + Block(dhidden, dhidden, rngs=rngs) for _ in range(depth - 2) ] - self.linear_out = Linear(dhidden, dout, rngs=rngs) + self.linear_out = Block(dhidden, dout, rngs=rngs) def __call__(self, x): self.count.value += 1 @@ -79,20 +89,16 @@ def main(argv): print(f'{mode=}, {total_steps=}, {batch_size=}, {width=}') - if mode not in ['nnx', 'jax']: - raise ValueError(f'Invalid mode: {mode}') - X = np.linspace(0, 1, 100)[:, None] Y = 0.8 * X**2 + 0.1 + np.random.normal(0, 0.1, size=X.shape) - model = MLP(din=1, dhidden=width, dout=1, depth=depth, rngs=nnx.Rngs(0)) - tx = optax.sgd(1e-3) - optimizer = nnx.Optimizer(model, tx) - t0 = time() - - if mode == 'nnx': + if mode == 'nnx' or mode == 'all': + model = MLP(din=1, dhidden=width, dout=1, depth=depth, rngs=nnx.Rngs(0)) + tx = optax.sgd(1e-3) + optimizer = nnx.Optimizer(model, tx) + t0 = time() - @nnx.jit + @nnx.jit(donate_argnums=(0, 1)) def train_step_nnx(model: MLP, optimizer: nnx.Optimizer, batch): x, y = batch @@ -103,26 +109,40 @@ def loss_fn(model: MLP): grads: nnx.State = nnx.grad(loss_fn)(model) optimizer.update(grads) - @nnx.jit + @nnx.jit(donate_argnums=0) def test_step_nnx(model: MLP, batch): x, y = batch y_pred = model(x) loss = jnp.mean((y - y_pred) ** 2) return {'loss': loss} + cached_train_step_nnx = nnx.cache_args(train_step_nnx, model, optimizer) + cached_test_step_nnx = nnx.cache_args(test_step_nnx, model) + for step, batch in enumerate(dataset(X, Y, batch_size)): - train_step_nnx(model, optimizer, batch) + cached_train_step_nnx(batch) if step % 1000 == 0: - logs = test_step_nnx(model, (X, Y)) - print(f"step: {step}, loss: {logs['loss']}") + logs = cached_test_step_nnx((X, Y)) if step >= total_steps - 1: break - else: - @jax.jit - def train_step_jax(graphdef, state, batch): + print('### NNX ###') + print(f"final loss: {logs['loss']}") + total_time = time() - t0 + print('total time:', total_time) + print(f'time per step: {total_time / total_steps * 1e6:.2f} µs') + print('times called:', model.count.value) + + if mode == 'jax' or mode == 'all': + model = MLP(din=1, dhidden=width, dout=1, depth=depth, rngs=nnx.Rngs(0)) + tx = optax.sgd(1e-3) + optimizer = nnx.Optimizer(model, tx) + t0 = time() + + @partial(jax.jit, donate_argnums=0) + def train_step_jax(state, batch): model, optimizer = nnx.merge(graphdef, state) x, y = batch @@ -135,8 +155,8 @@ def loss_fn(model: MLP): return nnx.state((model, optimizer)) - @jax.jit - def test_step_jax(graphdef, state, batch): + @partial(jax.jit, donate_argnums=0) + def test_step_jax(state, batch): model, optimizer = nnx.merge(graphdef, state) x, y = batch y_pred = model(x) @@ -147,21 +167,22 @@ def test_step_jax(graphdef, state, batch): graphdef, state = nnx.split((model, optimizer)) for step, batch in enumerate(dataset(X, Y, batch_size)): - state = train_step_jax(graphdef, state, batch) + state = train_step_jax(state, batch) if step % 1000 == 0: - state, logs = test_step_jax(graphdef, state, (X, Y)) - print(f"step: {step}, loss: {logs['loss']}") + state, logs = test_step_jax(state, (X, Y)) if step >= total_steps - 1: break model, optimizer = nnx.merge(graphdef, state) - total_time = time() - t0 - print('total time:', total_time) - print(f'time per step: {total_time / total_steps * 1e6:.2f} µs') - print('times called:', model.count.value) + print('### JAX ###') + print(f"final loss: {logs['loss']}") + total_time = time() - t0 + print('total time:', total_time) + print(f'time per step: {total_time / total_steps * 1e6:.2f} µs') + print('times called:', model.count.value) if __name__ == '__main__': diff --git a/docs_nnx/guides/checkpointing.ipynb b/docs_nnx/guides/checkpointing.ipynb index de6c7a279d..449f8a7755 100644 --- a/docs_nnx/guides/checkpointing.ipynb +++ b/docs_nnx/guides/checkpointing.ipynb @@ -88,7 +88,7 @@ { "data": { "text/html": [ - "
" + "
" ], "text/plain": [ "" @@ -100,7 +100,7 @@ { "data": { "text/html": [ - "
" + "
" ], "text/plain": [ "" @@ -153,7 +153,7 @@ { "data": { "text/html": [ - "
" + "
" ], "text/plain": [ "" @@ -173,14 +173,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "/Users/cris/repos/cristian/flax/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py:1136: UserWarning: Couldn't find sharding info under RestoreArgs. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file instead of directly from RestoreArgs. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.\n", + "/Users/ivyzheng/envs/flax-head/lib/python3.11/site-packages/orbax/checkpoint/type_handlers.py:1439: UserWarning: Couldn't find sharding info under RestoreArgs. Populating sharding info from sharding file. Please note restoration time will be slightly increased due to reading from file instead of directly from RestoreArgs. Note also that this option is unsafe when restoring on a different topology than the checkpoint was saved with.\n", " warnings.warn(\n" ] }, { "data": { "text/html": [ - "
" + "
" ], "text/plain": [ "" @@ -192,7 +192,7 @@ { "data": { "text/html": [ - "
" + "
" ], "text/plain": [ "" @@ -258,7 +258,7 @@ { "data": { "text/html": [ - "
" + "
" ], "text/plain": [ "" @@ -270,7 +270,7 @@ { "data": { "text/html": [ - "
" + "
" ], "text/plain": [ "" @@ -338,7 +338,7 @@ { "data": { "text/html": [ - "
" + "
" ], "text/plain": [ "" @@ -350,7 +350,7 @@ { "data": { "text/html": [ - "
" + "
" ], "text/plain": [ "" @@ -440,7 +440,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.16" + "version": "3.11.9" } }, "nbformat": 4, diff --git a/docs_nnx/guides/filters_guide.ipynb b/docs_nnx/guides/filters_guide.ipynb index fbcbc5fd11..a4dfabea97 100644 --- a/docs_nnx/guides/filters_guide.ipynb +++ b/docs_nnx/guides/filters_guide.ipynb @@ -5,17 +5,12 @@ "id": "95b08e64", "metadata": {}, "source": [ - "# Using Filters, grouping NNX variables \n", + "# Using Filters\n", "\n", - "Flax NNX uses [`Filter`s](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/filterlib.html) extensively as a way to create [`nnx.State`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/state.html#flax.nnx.State) groups in APIs, such as [`nnx.split`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/graph.html#flax.nnx.split), [`nnx.state()`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/graph.html#flax.nnx.state), and many of the [Flax NNX transformations (transforms)](https://flax.readthedocs.io/en/latest/guides/jax_and_nnx_transforms.html).\n", + "> **Attention**: This page relates to the new Flax NNX API.\n", "\n", - "In this guide you will learn how to:\n", - "\n", - "* Use [`Filter`s](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/filterlib.html) to group Flax NNX variables and states into subgroups;\n", - "* Understand relationships between types, such as [`nnx.Param`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/variables.html#flax.nnx.Param) or [`nnx.BatchStat`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/variables.html#flax.nnx.BatchStat), and [`Filter`s](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/filterlib.html);\n", - "* Express your `Filter`s flexibly with [`nnx.filterlib.Filter`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/filterlib.html) language.\n", - "\n", - "In the following example [`nnx.Param`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/variables.html#flax.nnx.Param) and [`nnx.BatchStat`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/variables.html#flax.nnx.BatchStat) are used as `Filter`s to split the model into two groups: one with the parameters and the other with the batch statistics:" + "Filters are used extensively in Flax NNX as a way to create `State` groups in APIs\n", + "such as `nnx.split`, `nnx.state`, and many of the Flax NNX transforms. For example:" ] }, { @@ -64,7 +59,11 @@ "id": "8f77e99a", "metadata": {}, "source": [ - "Let's dive deeper into `Filter`s." + "Here `nnx.Param` and `nnx.BatchStat` are used as Filters to split the model into two groups: one with the parameters and the other with the batch statistics. However, this begs the following questions:\n", + "\n", + "* What is a Filter?\n", + "* Why are types, such as `Param` or `BatchStat`, Filters?\n", + "* How is `State` grouped / filtered?" ] }, { @@ -72,25 +71,20 @@ "id": "a0413d64", "metadata": {}, "source": [ - "## The `Filter` Protocol\n", + "## The Filter Protocol\n", "\n", - "In general, Flax `Filter`s are predicate functions of the form:\n", + "In general Filter are predicate functions of the form:\n", "\n", "```python\n", "\n", "(path: tuple[Key, ...], value: Any) -> bool\n", "\n", "```\n", + "where `Key` is a hashable and comparable type, `path` is a tuple of `Key`s representing the path to the value in a nested structure, and `value` is the value at the path. The function returns `True` if the value should be included in the group and `False` otherwise.\n", "\n", - "where:\n", - "\n", - "- `Key` is a hashable and comparable type;\n", - "- `path` is a tuple of `Key`s representing the path to the value in a nested structure; and\n", - "- `value` is the value at the path.\n", - "\n", - "The function returns `True` if the value should be included in the group, and `False` otherwise.\n", - "\n", - "Types are not functions of this form. They are treated as `Filter`s because, as you will learn in the next section, types and some other literals are converted to _predicates_. For example, [`nnx.Param`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/variables.html#flax.nnx.Param) is roughly converted to a predicate like this:" + "Types are obviously not functions of this form, so the reason why they are treated as Filters\n", + "is because, as we will see next, types and some other literals are converted to predicates. For example,\n", + "`Param` is roughly converted to a predicate like this:" ] }, { @@ -123,7 +117,9 @@ "id": "a8a2641e", "metadata": {}, "source": [ - "Such function matches any value that is an instance of [`nnx.Param`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/variables.html#flax.nnx.Param) or any value that has a `type` attribute that is a subclass of [`nnx.Param`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/variables.html#flax.nnx.Param). Internally Flax NNX uses `OfType` which defines a callable of this form for a given type:" + "Such function matches any value that is an instance of `Param` or any value that has a\n", + "`type` attribute that is a subclass of `Param`. Internally Flax NNX uses `OfType` which\n", + "defines a callable of this form for a given type:" ] }, { @@ -153,11 +149,14 @@ "id": "87c06e39", "metadata": {}, "source": [ - "## The `Filter` DSL\n", + "## The Filter DSL\n", "\n", - "Flax NNX exposes a small domain specific language ([DSL](https://en.wikipedia.org/wiki/Domain-specific_language)), formalized as the [`nnx.filterlib.Filter`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/filterlib.html) type. This means users don't have to create functions like in the previous section.\n", + "To avoid users having to create these functions, Flax NNX exposes a small DSL, formalized\n", + "as the `nnx.filterlib.Filter` type, which lets users pass types, booleans, ellipsis,\n", + "tuples/lists, etc, and converts them to the appropriate predicate internally.\n", "\n", - "Here is a list of all the callable `Filter`s included in Flax NNX, and their corresponding DSL literals (when available):\n", + "Here is a list of all the callable Filters included in Flax NNX and their DSL literals\n", + "(when available):\n", "\n", "\n", "| Literal | Callable | Description |\n", @@ -171,14 +170,10 @@ "| | `All(*filters)` | Matches values that match all of the inner `filters` |\n", "| | `Not(filter)` | Matches values that do not match the inner `filter` |\n", "\n", - "\n", - "Let's check out the DSL in action by using [`nnx.vmap`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/transforms.html#flax.nnx.vmap) as an example. Consider the following:\n", - "\n", - "1) You want to vectorize all parameters;\n", - "2) Apply `'dropout'` `Rng(Keys|Counts)` on the `0`th axis; and\n", - "3) Broadcast the rest.\n", - "\n", - "To do this, you can use the following `Filter`s to define a `nnx.StateAxes` object that you can pass to `nnx.vmap`'s `in_axes` to specify how the `model`'s various sub-states should be vectorized:" + "Let see the DSL in action with a `nnx.vmap` example. Lets say we want vectorized all parameters\n", + "and `dropout` Rng(Keys|Counts) on the 0th axis, and broadcasted the rest. To do so we can\n", + "use the following filters to define a `nnx.StateAxes` object that we can pass to `nnx.vmap`'s `in_axes`\n", + "to specify how `model`'s various substates should be vectorized:" ] }, { @@ -200,9 +195,10 @@ "id": "bd60f0e1", "metadata": {}, "source": [ - "Here `(nnx.Param, 'dropout')` expands to `Any(OfType(nnx.Param), WithTag('dropout'))` and `...` expands to `Everything()`.\n", + "Here `(nnx.Param, 'dropout')` expands to `Any(OfType(nnx.Param), WithTag('dropout'))` and `...`\n", + "expands to `Everything()`.\n", "\n", - "If you wish to manually convert literal into a predicate, you can use [`nnx.filterlib.to_predicate`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/filterlib.html#flax.nnx.filterlib.to_predicate):" + "If you wish to manually convert literal into a predicate to can use `nnx.filterlib.to_predicate`:" ] }, { @@ -239,15 +235,15 @@ "id": "db9b4cf3", "metadata": {}, "source": [ - "## Grouping `State`s\n", + "## Grouping States\n", "\n", - "With the knowledge of `Filter`s from previous sections at hand, let's learn how to roughly implement [`nnx.split`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/graph.html#flax.nnx.split). Here are the key ideas:\n", + "With the knowledge of Filters at hand, let's see how `nnx.split` is roughly implemented. Key ideas:\n", "\n", - "* Use `nnx.graph.flatten` to get the [`GraphDef`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/graph.html#flax.nnx.GraphDef) and [`nnx.State`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/state.html#flax.nnx.State) representation of the node.\n", - "* Convert all the `Filter`s to predicates.\n", + "* Use `nnx.graph.flatten` to get the `GraphDef` and `State` representation of the node.\n", + "* Convert all the filters to predicates.\n", "* Use `State.flat_state` to get the flat representation of the state.\n", "* Traverse all the `(path, value)` pairs in the flat state and group them according to the predicates.\n", - "* Use `State.from_flat_state` to convert the flat states to nested [`nnx.State`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/state.html#flax.nnx.State)s." + "* Use `State.from_flat_state` to convert the flat states to nested `State`s." ] }, { @@ -297,7 +293,7 @@ " )\n", " return graphdef, *states\n", "\n", - "# Let's test it.\n", + "# lets test it...\n", "foo = Foo()\n", "\n", "graphdef, params, batch_stats = split(foo, nnx.Param, nnx.BatchStat)\n", @@ -311,14 +307,12 @@ "id": "7b3aeac8", "metadata": {}, "source": [ - "**Note:*** It's very important to know that **filtering is order-dependent**. The first `Filter` that matches a value will keep it, and therefore you should place more specific `Filter`s before more general `Filter`s.\n", - "\n", - "For example, as demonstrated below, if you:\n", - "\n", - "1) Create a `SpecialParam` type that is a subclass of [`nnx.Param`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/variables.html#flax.nnx.Param), and a `Bar` object (subclassing [`nnx.Module`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/module.html)) that contains both types of parameters; and\n", - "2) Try to split the [`nnx.Param`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/variables.html#flax.nnx.Param)s before the `SpecialParam`s\n", - "\n", - "then all the values will be placed in the [`nnx.Param`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/variables.html#flax.nnx.Param) group, and the `SpecialParam` group will be empty because all `SpecialParam`s are also [`nnx.Param`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/variables.html#flax.nnx.Param)s:" + "One very important thing to note is that **filtering is order-dependent**. The first filter that\n", + "matches a value will keep it, therefore you should place more specific filters before more general\n", + "filters. For example if we create a `SpecialParam` type that is a subclass of `Param`, and a `Bar`\n", + "object that contains both types of parameters, if we try to split the `Param`s before the\n", + "`SpecialParam`s then all the values will be placed in the `Param` group and the `SpecialParam` group\n", + "will be empty because all `SpecialParam`s are also `Param`s:" ] }, { @@ -366,7 +360,7 @@ "id": "a9f0b7b8", "metadata": {}, "source": [ - "And reversing the order will ensure that the `SpecialParam` are captured first:" + "Reversing the order will make sure that the `SpecialParam` are captured first" ] }, { diff --git a/docs_nnx/guides/filters_guide.md b/docs_nnx/guides/filters_guide.md index 88a25a6a5a..dcd414d76a 100644 --- a/docs_nnx/guides/filters_guide.md +++ b/docs_nnx/guides/filters_guide.md @@ -8,17 +8,12 @@ jupytext: jupytext_version: 1.13.8 --- -# Using Filters, grouping NNX variables +# Using Filters -Flax NNX uses [`Filter`s](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/filterlib.html) extensively as a way to create [`nnx.State`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/state.html#flax.nnx.State) groups in APIs, such as [`nnx.split`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/graph.html#flax.nnx.split), [`nnx.state()`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/graph.html#flax.nnx.state), and many of the [Flax NNX transformations (transforms)](https://flax.readthedocs.io/en/latest/guides/jax_and_nnx_transforms.html). +> **Attention**: This page relates to the new Flax NNX API. -In this guide you will learn how to: - -* Use [`Filter`s](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/filterlib.html) to group Flax NNX variables and states into subgroups; -* Understand relationships between types, such as [`nnx.Param`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/variables.html#flax.nnx.Param) or [`nnx.BatchStat`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/variables.html#flax.nnx.BatchStat), and [`Filter`s](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/filterlib.html); -* Express your `Filter`s flexibly with [`nnx.filterlib.Filter`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/filterlib.html) language. - -In the following example [`nnx.Param`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/variables.html#flax.nnx.Param) and [`nnx.BatchStat`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/variables.html#flax.nnx.BatchStat) are used as `Filter`s to split the model into two groups: one with the parameters and the other with the batch statistics: +Filters are used extensively in Flax NNX as a way to create `State` groups in APIs +such as `nnx.split`, `nnx.state`, and many of the Flax NNX transforms. For example: ```{code-cell} ipython3 from flax import nnx @@ -36,29 +31,28 @@ print(f'{params = }') print(f'{batch_stats = }') ``` -Let's dive deeper into `Filter`s. +Here `nnx.Param` and `nnx.BatchStat` are used as Filters to split the model into two groups: one with the parameters and the other with the batch statistics. However, this begs the following questions: + +* What is a Filter? +* Why are types, such as `Param` or `BatchStat`, Filters? +* How is `State` grouped / filtered? +++ -## The `Filter` Protocol +## The Filter Protocol -In general, Flax `Filter`s are predicate functions of the form: +In general Filter are predicate functions of the form: ```python (path: tuple[Key, ...], value: Any) -> bool ``` +where `Key` is a hashable and comparable type, `path` is a tuple of `Key`s representing the path to the value in a nested structure, and `value` is the value at the path. The function returns `True` if the value should be included in the group and `False` otherwise. -where: - -- `Key` is a hashable and comparable type; -- `path` is a tuple of `Key`s representing the path to the value in a nested structure; and -- `value` is the value at the path. - -The function returns `True` if the value should be included in the group, and `False` otherwise. - -Types are not functions of this form. They are treated as `Filter`s because, as you will learn in the next section, types and some other literals are converted to _predicates_. For example, [`nnx.Param`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/variables.html#flax.nnx.Param) is roughly converted to a predicate like this: +Types are obviously not functions of this form, so the reason why they are treated as Filters +is because, as we will see next, types and some other literals are converted to predicates. For example, +`Param` is roughly converted to a predicate like this: ```{code-cell} ipython3 def is_param(path, value) -> bool: @@ -70,7 +64,9 @@ print(f'{is_param((), nnx.Param(0)) = }') print(f'{is_param((), nnx.VariableState(type=nnx.Param, value=0)) = }') ``` -Such function matches any value that is an instance of [`nnx.Param`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/variables.html#flax.nnx.Param) or any value that has a `type` attribute that is a subclass of [`nnx.Param`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/variables.html#flax.nnx.Param). Internally Flax NNX uses `OfType` which defines a callable of this form for a given type: +Such function matches any value that is an instance of `Param` or any value that has a +`type` attribute that is a subclass of `Param`. Internally Flax NNX uses `OfType` which +defines a callable of this form for a given type: ```{code-cell} ipython3 is_param = nnx.OfType(nnx.Param) @@ -79,11 +75,14 @@ print(f'{is_param((), nnx.Param(0)) = }') print(f'{is_param((), nnx.VariableState(type=nnx.Param, value=0)) = }') ``` -## The `Filter` DSL +## The Filter DSL -Flax NNX exposes a small domain specific language ([DSL](https://en.wikipedia.org/wiki/Domain-specific_language)), formalized as the [`nnx.filterlib.Filter`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/filterlib.html) type. This means users don't have to create functions like in the previous section. +To avoid users having to create these functions, Flax NNX exposes a small DSL, formalized +as the `nnx.filterlib.Filter` type, which lets users pass types, booleans, ellipsis, +tuples/lists, etc, and converts them to the appropriate predicate internally. -Here is a list of all the callable `Filter`s included in Flax NNX, and their corresponding DSL literals (when available): +Here is a list of all the callable Filters included in Flax NNX and their DSL literals +(when available): | Literal | Callable | Description | @@ -97,14 +96,10 @@ Here is a list of all the callable `Filter`s included in Flax NNX, and their cor | | `All(*filters)` | Matches values that match all of the inner `filters` | | | `Not(filter)` | Matches values that do not match the inner `filter` | - -Let's check out the DSL in action by using [`nnx.vmap`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/transforms.html#flax.nnx.vmap) as an example. Consider the following: - -1) You want to vectorize all parameters; -2) Apply `'dropout'` `Rng(Keys|Counts)` on the `0`th axis; and -3) Broadcast the rest. - -To do this, you can use the following `Filter`s to define a `nnx.StateAxes` object that you can pass to `nnx.vmap`'s `in_axes` to specify how the `model`'s various sub-states should be vectorized: +Let see the DSL in action with a `nnx.vmap` example. Lets say we want vectorized all parameters +and `dropout` Rng(Keys|Counts) on the 0th axis, and broadcasted the rest. To do so we can +use the following filters to define a `nnx.StateAxes` object that we can pass to `nnx.vmap`'s `in_axes` +to specify how `model`'s various substates should be vectorized: ```{code-cell} ipython3 state_axes = nnx.StateAxes({(nnx.Param, 'dropout'): 0, ...: None}) @@ -114,9 +109,10 @@ def forward(model, x): ... ``` -Here `(nnx.Param, 'dropout')` expands to `Any(OfType(nnx.Param), WithTag('dropout'))` and `...` expands to `Everything()`. +Here `(nnx.Param, 'dropout')` expands to `Any(OfType(nnx.Param), WithTag('dropout'))` and `...` +expands to `Everything()`. -If you wish to manually convert literal into a predicate, you can use [`nnx.filterlib.to_predicate`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/filterlib.html#flax.nnx.filterlib.to_predicate): +If you wish to manually convert literal into a predicate to can use `nnx.filterlib.to_predicate`: ```{code-cell} ipython3 is_param = nnx.filterlib.to_predicate(nnx.Param) @@ -130,15 +126,15 @@ print(f'{nothing = }') print(f'{params_or_dropout = }') ``` -## Grouping `State`s +## Grouping States -With the knowledge of `Filter`s from previous sections at hand, let's learn how to roughly implement [`nnx.split`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/graph.html#flax.nnx.split). Here are the key ideas: +With the knowledge of Filters at hand, let's see how `nnx.split` is roughly implemented. Key ideas: -* Use `nnx.graph.flatten` to get the [`GraphDef`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/graph.html#flax.nnx.GraphDef) and [`nnx.State`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/state.html#flax.nnx.State) representation of the node. -* Convert all the `Filter`s to predicates. +* Use `nnx.graph.flatten` to get the `GraphDef` and `State` representation of the node. +* Convert all the filters to predicates. * Use `State.flat_state` to get the flat representation of the state. * Traverse all the `(path, value)` pairs in the flat state and group them according to the predicates. -* Use `State.from_flat_state` to convert the flat states to nested [`nnx.State`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/state.html#flax.nnx.State)s. +* Use `State.from_flat_state` to convert the flat states to nested `State`s. ```{code-cell} ipython3 from typing import Any @@ -162,7 +158,7 @@ def split(node, *filters): ) return graphdef, *states -# Let's test it. +# lets test it... foo = Foo() graphdef, params, batch_stats = split(foo, nnx.Param, nnx.BatchStat) @@ -171,14 +167,12 @@ print(f'{params = }') print(f'{batch_stats = }') ``` -**Note:*** It's very important to know that **filtering is order-dependent**. The first `Filter` that matches a value will keep it, and therefore you should place more specific `Filter`s before more general `Filter`s. - -For example, as demonstrated below, if you: - -1) Create a `SpecialParam` type that is a subclass of [`nnx.Param`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/variables.html#flax.nnx.Param), and a `Bar` object (subclassing [`nnx.Module`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/module.html)) that contains both types of parameters; and -2) Try to split the [`nnx.Param`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/variables.html#flax.nnx.Param)s before the `SpecialParam`s - -then all the values will be placed in the [`nnx.Param`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/variables.html#flax.nnx.Param) group, and the `SpecialParam` group will be empty because all `SpecialParam`s are also [`nnx.Param`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/variables.html#flax.nnx.Param)s: +One very important thing to note is that **filtering is order-dependent**. The first filter that +matches a value will keep it, therefore you should place more specific filters before more general +filters. For example if we create a `SpecialParam` type that is a subclass of `Param`, and a `Bar` +object that contains both types of parameters, if we try to split the `Param`s before the +`SpecialParam`s then all the values will be placed in the `Param` group and the `SpecialParam` group +will be empty because all `SpecialParam`s are also `Param`s: ```{code-cell} ipython3 class SpecialParam(nnx.Param): @@ -196,7 +190,7 @@ print(f'{params = }') print(f'{special_params = }') ``` -And reversing the order will ensure that the `SpecialParam` are captured first: +Reversing the order will make sure that the `SpecialParam` are captured first ```{code-cell} ipython3 graphdef, special_params, params = split(bar, SpecialParam, nnx.Param) # correct! diff --git a/docs_nnx/mnist_tutorial.ipynb b/docs_nnx/mnist_tutorial.ipynb index bba6fb0001..a1aa4eae89 100644 --- a/docs_nnx/mnist_tutorial.ipynb +++ b/docs_nnx/mnist_tutorial.ipynb @@ -56,7 +56,19 @@ "execution_count": 2, "id": "4", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/google/home/cgarciae/flax/.venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n", + "2024-07-10 15:24:11.227958: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", + "2024-07-10 15:24:12.227896: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" + ] + } + ], "source": [ "import tensorflow_datasets as tfds # TFDS to download MNIST.\n", "import tensorflow as tf # TensorFlow / `tf.data` operations.\n", @@ -110,19 +122,7 @@ { "data": { "text/html": [ - "
" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
" + "
(Loading...)
" ], "text/plain": [ "" @@ -180,21 +180,22 @@ "outputs": [ { "data": { + "text/html": [ + "
(Loading...)
" + ], "text/plain": [ - "Array([[-0.06820839, -0.14743432, 0.00265857, -0.2173656 , 0.16673787,\n", - " -0.00923921, -0.06636689, 0.28341877, 0.33754364, -0.20142877]], dtype=float32)" + "" ] }, - "execution_count": 4, "metadata": {}, - "output_type": "execute_result" + "output_type": "display_data" } ], "source": [ "import jax.numpy as jnp # JAX NumPy\n", "\n", "y = model(jnp.ones((1, 28, 28, 1)))\n", - "y" + "nnx.display(y)" ] }, { @@ -216,19 +217,7 @@ { "data": { "text/html": [ - "
" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
" + "
(Loading...)
" ], "text/plain": [ "" @@ -326,20 +315,105 @@ }, "outputs": [ { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABMYAAAHDCAYAAADP+BbYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAADKnklEQVR4nOzdd3hUZfrG8e+kN1IgjRQIhN4SakTsRhDUpYvoSnHX/YniFnRdWRERV1FXWVwbLi4WREVpYsNFFBWFhN57S4E0SgLpmZnfHycZiIAmkORkkvtzXXM5c/LOmWcYlZN73vd5LXa73Y6IiIiIiIiIiEgj42J2ASIiIiIiIiIiImZQMCYiIiIiIiIiIo2SgjEREREREREREWmUFIyJiIiIiIiIiEijpGBMREREREREREQaJQVjIiIiIiIiIiLSKCkYExERERERERGRRknBmIiIiIiIiIiINEoKxkREREREREREpFFSMCYiIiIiIiIiIo2SgjERqffefvttLBYL69evN7sUERERESn32muvYbFYSEhIMLsUEZFLpmBMREREREREqm3+/PnExMSQnJzM/v37zS5HROSSKBgTERERERGRajl06BA//fQTM2fOJCQkhPnz55td0gXl5+ebXYKI1HMKxkSkQdi0aRMDBw7E398fPz8/brzxRtauXVtpTGlpKU8++SRt27bFy8uLZs2acdVVV7FixQrHmIyMDMaPH09UVBSenp40b96cwYMHc/jw4Tp+RyIiIiL11/z58wkKCuKWW25hxIgRFwzGTp06xV/+8hdiYmLw9PQkKiqKMWPGkJOT4xhTVFTEtGnTaNeuHV5eXjRv3pxhw4Zx4MABAFatWoXFYmHVqlWVzn348GEsFgtvv/2249i4cePw8/PjwIEDDBo0iCZNmnDXXXcB8MMPPzBy5EhatGiBp6cn0dHR/OUvf6GwsPC8unfv3s3tt99OSEgI3t7etG/fnsceewyAb7/9FovFwpIlS8573vvvv4/FYmHNmjXV/vMUEfO4mV2AiMjl2rFjB1dffTX+/v488sgjuLu788Ybb3Ddddfx3XffOfpeTJs2jRkzZvD73/+ePn36kJeXx/r169m4cSM33XQTAMOHD2fHjh08+OCDxMTEkJWVxYoVK0hJSSEmJsbEdykiIiJSf8yfP59hw4bh4eHB6NGjef3111m3bh29e/cG4MyZM1x99dXs2rWLe+65hx49epCTk8OyZctIS0sjODgYq9XKrbfeysqVK7njjjv405/+xOnTp1mxYgXbt28nNja22nWVlZUxYMAArrrqKl544QV8fHwA+PjjjykoKGDChAk0a9aM5ORkXn75ZdLS0vj4448dz9+6dStXX3017u7u/OEPfyAmJoYDBw7w6aef8vTTT3PdddcRHR3N/PnzGTp06Hl/JrGxsfTt2/cy/mRFpM7ZRUTqubfeessO2NetW3fBnw8ZMsTu4eFhP3DggOPY0aNH7U2aNLFfc801jmNxcXH2W2655aKvc/LkSTtg/+c//1lzxYuIiIg0MOvXr7cD9hUrVtjtdrvdZrPZo6Ki7H/6058cY6ZOnWoH7IsXLz7v+TabzW632+1z5861A/aZM2dedMy3335rB+zffvttpZ8fOnTIDtjfeustx7GxY8faAfujjz563vkKCgrOOzZjxgy7xWKxHzlyxHHsmmuusTdp0qTSsXPrsdvt9smTJ9s9PT3tp06dchzLysqyu7m52Z944onzXkdE6jctpRQRp2a1Wvnf//7HkCFDaN26teN48+bNufPOO1m9ejV5eXkABAYGsmPHDvbt23fBc3l7e+Ph4cGqVas4efJkndQvIiIi4mzmz59PWFgY119/PQAWi4VRo0bx4YcfYrVaAVi0aBFxcXHnzaqqGF8xJjg4mAcffPCiYy7FhAkTzjvm7e3tuJ+fn09OTg5XXnkldrudTZs2AZCdnc3333/PPffcQ4sWLS5az5gxYyguLmbhwoWOYwsWLKCsrIzf/va3l1y3iJhDwZiIOLXs7GwKCgpo3779eT/r2LEjNpuN1NRUAKZPn86pU6do164dXbt25a9//Stbt251jPf09OS5557jyy+/JCwsjGuuuYbnn3+ejIyMOns/IiIiIvWZ1Wrlww8/5Prrr+fQoUPs37+f/fv3k5CQQGZmJitXrgTgwIEDdOnS5RfPdeDAAdq3b4+bW811+HFzcyMqKuq84ykpKYwbN46mTZvi5+dHSEgI1157LQC5ubkAHDx4EOBX6+7QoQO9e/eu1Fdt/vz5XHHFFbRp06am3oqI1BEFYyLSaFxzzTUcOHCAuXPn0qVLF95880169OjBm2++6Rjz5z//mb179zJjxgy8vLx4/PHH6dixo+ObRBEREZHG7JtvvuHYsWN8+OGHtG3b1nG7/fbbAWp8d8qLzRyrmJn2c56enri4uJw39qabbuLzzz/nb3/7G0uXLmXFihWOxv02m63adY0ZM4bvvvuOtLQ0Dhw4wNq1azVbTMRJqfm+iDi1kJAQfHx82LNnz3k/2717Ny4uLkRHRzuONW3alPHjxzN+/HjOnDnDNddcw7Rp0/j973/vGBMbG8tDDz3EQw89xL59+4iPj+fFF1/kvffeq5P3JCIiIlJfzZ8/n9DQUF599dXzfrZ48WKWLFnC7NmziY2NZfv27b94rtjYWJKSkigtLcXd3f2CY4KCggBjh8tzHTlypMo1b9u2jb179/LOO+8wZswYx/FzdyYHHG05fq1ugDvuuINJkybxwQcfUFhYiLu7O6NGjapyTSJSf2jGmIg4NVdXV/r3788nn3zC4cOHHcczMzN5//33ueqqq/D39wfg+PHjlZ7r5+dHmzZtKC4uBqCgoICioqJKY2JjY2nSpIljjIiIiEhjVVhYyOLFi7n11lsZMWLEebeJEydy+vRpli1bxvDhw9myZQtLliw57zx2ux0wdgPPycnhlVdeueiYli1b4urqyvfff1/p56+99lqV63Z1da10zor7L730UqVxISEhXHPNNcydO5eUlJQL1lMhODiYgQMH8t577zF//nxuvvlmgoODq1yTiNQfmjEmIk5j7ty5LF++/Lzj06ZNY8WKFVx11VXcf//9uLm58cYbb1BcXMzzzz/vGNepUyeuu+46evbsSdOmTVm/fj0LFy5k4sSJAOzdu5cbb7yR22+/nU6dOuHm5saSJUvIzMzkjjvuqLP3KSIiIlIfLVu2jNOnT/Ob3/zmgj+/4oorCAkJYf78+bz//vssXLiQkSNHcs8999CzZ09OnDjBsmXLmD17NnFxcYwZM4Z3332XSZMmkZyczNVXX01+fj5ff/01999/P4MHDyYgIICRI0fy8ssvY7FYiI2N5bPPPiMrK6vKdXfo0IHY2Fgefvhh0tPT8ff3Z9GiRRfcbOnf//43V111FT169OAPf/gDrVq14vDhw3z++eds3ry50tgxY8YwYsQIAJ566qmq/0GKSP1i5paYIiJV8dZbb9mBi95SU1PtGzdutA8YMMDu5+dn9/HxsV9//fX2n376qdJ5/vGPf9j79OljDwwMtHt7e9s7dOhgf/rpp+0lJSV2u91uz8nJsT/wwAP2Dh062H19fe0BAQH2hIQE+0cffWTG2xYRERGpV2677Ta7l5eXPT8//6Jjxo0bZ3d3d7fn5OTYjx8/bp84caI9MjLS7uHhYY+KirKPHTvWnpOT4xhfUFBgf+yxx+ytWrWyu7u728PDw+0jRoywHzhwwDEmOzvbPnz4cLuPj489KCjI/n//93/27du32wH7W2+95Rg3duxYu6+v7wXr2rlzpz0xMdHu5+dnDw4Ott977732LVu2nHcOu91u3759u33o0KH2wMBAu5eXl719+/b2xx9//LxzFhcX24OCguwBAQH2wsLCKv4pikh9Y7HbfzYnVERERERERER+UVlZGREREdx2223897//NbscEblE6jEmIiIiIiIiUk1Lly4lOzu7UkN/EXE+mjEmIiIiIiIiUkVJSUls3bqVp556iuDgYDZu3Gh2SSJyGTRjTERERERERKSKXn/9dSZMmEBoaCjvvvuu2eWIyGXSjDEREREREREREWmUNGNMREREREREREQaJQVjIiIiIiIiIiLSKLmZXUBNsNlsHD16lCZNmmCxWMwuR0RERJyE3W7n9OnTRERE4OKi7wvrK13riYiISHVV9TqvQQRjR48eJTo62uwyRERExEmlpqYSFRVldhlyEbrWExERkUv1a9d5DSIYa9KkCWC8WX9/f5OrEREREWeRl5dHdHS041pC6idd64mIiEh1VfU6r0EEYxVT6v39/XWxJCIiItWm5Xn1m671RERE5FL92nWemmmIiIiIiIiIiEijpGBMREREREREREQaJQVjIiIiIiIiIiLSKDWIHmMiIiK1xWq1UlpaanYZconc3d1xdXU1uwwRERERqacUjImIiFyA3W4nIyODU6dOmV2KXKbAwEDCw8PVYF9EREREzqNgTERE5AIqQrHQ0FB8fHwUqjghu91OQUEBWVlZADRv3tzkikRERESkvlEwJiIi8jNWq9URijVr1szscuQyeHt7A5CVlUVoaKiWVYqIiIhIJWq+LyIi8jMVPcV8fHxMrkRqQsXnqF5xIiIiIvJzCsZEREQuQssnGwZ9jiIiIiJyMQrGRERERERERESkUVIwJiIiIhcUExPDrFmzauRcq1atwmKxaJdPEREREalX1HxfRESkAbnuuuuIj4+vkUBr3bp1+Pr6Xn5RIiIiIiL1lIIxERGRRsRut2O1WnFz+/VLgJCQkDqoSERERETEPFpKWQVFpVY+2ZxORm6R2aWIiIhc1Lhx4/juu+946aWXsFgsWCwW3n77bSwWC19++SU9e/bE09OT1atXc+DAAQYPHkxYWBh+fn707t2br7/+utL5fr6U0mKx8OabbzJ06FB8fHxo27Yty5Ytu+R6Fy1aROfOnfH09CQmJoYXX3yx0s9fe+012rZti5eXF2FhYYwYMcLxs4ULF9K1a1e8vb1p1qwZiYmJ5OfnX3ItIiIiIlLHsnbBrk/NrkIzxqriwQ82sWJnJn9ObMufE9uZXY6IiNQxu91OYanVlNf2dnet8q6KL730Env37qVLly5Mnz4dgB07dgDw6KOP8sILL9C6dWuCgoJITU1l0KBBPP3003h6evLuu+9y2223sWfPHlq0aHHR13jyySd5/vnn+ec//8nLL7/MXXfdxZEjR2jatGm13teGDRu4/fbbmTZtGqNGjeKnn37i/vvvp1mzZowbN47169fzxz/+kXnz5nHllVdy4sQJfvjhBwCOHTvG6NGjef755xk6dCinT5/mhx9+wG63V6sGEREREaljNhvs/xrWvgYHvwWvQIi9ATzMa9+hYKwKbouLYMXOTBasS2Xi9W1wc9VEOxGRxqSw1EqnqV+Z8to7pw/Ax6Nqf10HBATg4eGBj48P4eHhAOzevRuA6dOnc9NNNznGNm3alLi4OMfjp556iiVLlrBs2TImTpx40dcYN24co0ePBuCZZ57h3//+N8nJydx8883Vel8zZ87kxhtv5PHHHwegXbt27Ny5k3/+85+MGzeOlJQUfH19ufXWW2nSpAktW7ake/fugBGMlZWVMWzYMFq2bAlA165dq/X6IiIiIlKHSvJh8/uQNBuO7zeOWVyg1TVQlGtqMKaEpwoGdA4jyMedY7lFfLc32+xyREREqq1Xr16VHp85c4aHH36Yjh07EhgYiJ+fH7t27SIlJeUXz9OtWzfHfV9fX/z9/cnKyqp2Pbt27aJfv36VjvXr1499+/ZhtVq56aabaNmyJa1bt+buu+9m/vz5FBQUABAXF8eNN95I165dGTlyJHPmzOHkyZPVrkFEREREatmpVFgxFWZ2hC8eNkIxzwDoOxH+uBlGzQP/CFNL1IyxKvB0c2VEzyjm/HCID5JTuLFjmNkliYhIHfJ2d2Xn9AGmvXZN+Pnukg8//DArVqzghRdeoE2bNnh7ezNixAhKSkp+8Tzu7u6VHlssFmw2W43UeK4mTZqwceNGVq1axf/+9z+mTp3KtGnTWLduHYGBgaxYsYKffvqJ//3vf7z88ss89thjJCUl0apVqxqvRURERESqwW6HtHXGcsmdy8Be3pKkaWtImADxo8Gzibk1nkPBWBXd0acFc344xDe7sziWW0jzAG+zSxIRkTpisViqvJzRbB4eHlitv94P7ccff2TcuHEMHToUMGaQHT58uJarO6tjx478+OOP59XUrl07XF2NMNDNzY3ExEQSExN54oknCAwM5JtvvmHYsGFYLBb69etHv379mDp1Ki1btmTJkiVMmjSpzt6DiIiIiJzDWgo7PzECsfQNZ4+3uhauuB/a9geX+rdw0Tmu8uuB2BA/Elo1JenQCT5al8afEtuaXZKIiMh5YmJiSEpK4vDhw/j5+V10Nlfbtm1ZvHgxt912GxaLhccff7xWZn5dzEMPPUTv3r156qmnGDVqFGvWrOGVV17htddeA+Czzz7j4MGDXHPNNQQFBfHFF19gs9lo3749SUlJrFy5kv79+xMaGkpSUhLZ2dl07NixzuoXERERkXIFJ2DDW5A8B04fM465ekK32+GKCRDW2dz6fkX9i+rqsTsTjF26FqxLwWrTzlciIlL/PPzww7i6utKpUydCQkIu2jNs5syZBAUFceWVV3LbbbcxYMAAevToUWd19ujRg48++ogPP/yQLl26MHXqVKZPn864ceMACAwMZPHixdxwww107NiR2bNn88EHH9C5c2f8/f35/vvvGTRoEO3atWPKlCm8+OKLDBw4sM7qFxEREWn0snbDp38y+oetnG6EYn5hcP0UmLQTBr9S70MxAIu9AextnpeXR0BAALm5ufj7+9fa6xSVWuk7YyUnC0qZO64XN3RQrzERkYaoqKiIQ4cO0apVK7y8vMwuRy7TL32edXUNIZdHn5OINFhlxWBxBVct5hInYbPBgZXGcskD35w93jwOrngAOg8FNw/z6jtHVa8fLmnG2KuvvkpMTAxeXl4kJCSQnJx80bGLFy+mV69eBAYG4uvrS3x8PPPmzas0xm63M3XqVJo3b463tzeJiYns27fvUkqrVV7urgzvEQXA+0mpJlcjIiIiIiIiTsduh0Pfw0dj4ZkIeLEdfP4QpCQZPxOpj0ryYd2b8GofmD/CCMUsLtDxNzB+OfzhO4gbVW9CseqodjC2YMECJk2axBNPPMHGjRuJi4tjwIABF92qvWnTpjz22GOsWbOGrVu3Mn78eMaPH89XX33lGPP888/z73//m9mzZ5OUlISvry8DBgygqKjo0t9ZLbmjj7Gc8pvdmWTk1r/6REREzHDffffh5+d3wdt9991ndnkiIiLmKzwFa183goV3boOdS8FWBgXHjcBhbn94qRt8/SRk7TK7WhHDqVRYMdVYLvn5Q3B8H3j6Q9+J8MfNMGoetOwLFovZlV6yai+lTEhIoHfv3rzyyisA2Gw2oqOjefDBB3n00UerdI4ePXpwyy238NRTT2G324mIiOChhx7i4YcfBiA3N5ewsDDefvtt7rjjjl89X11Pr7/9jTUkHzrBpJva8ccb1YRfRKSh0VLK6svKyiIvL++CP/P39yc0NLSOKzpLSymdnz4nEXFqRzcZwde2RVBWaBzz8DMak/ccB/nZsG0h7PoUSs6cfV5YF+g6ArqMgMBoU0qXRspuh7R1xnLJncvAXr7jedPWkDAB4keDZxNza6yCql4/VGshc0lJCRs2bGDy5MmOYy4uLiQmJrJmzZpffb7dbuebb75hz549PPfccwAcOnSIjIwMEhMTHeMCAgJISEhgzZo1FwzGiouLKS4udjy+2IV4bbmzTwuSD51gwbpUHri+Da4uzpuMioiI1ITQ0FBTwy8REZF6paQAdiyGdf+FoxvPHg/tDL3vgW6jKgcLbRLhlpmwd7kRku37H2RuN25fT4MWVxohWach4Nusrt+NNBbWUtj5iRGIpW84e7zVNXDF/dB2ALg0vD0cqxWM5eTkYLVaCQur3HQ+LCyM3bt3X/R5ubm5REZGUlxcjKurK6+99ho33XQTABkZGY5z/PycFT/7uRkzZvDkk09Wp/QadXOXcAI/dSf9VCHf78vm+vb6RUBERERERKTRy9kH6+fC5vlQlGscc/WAToOh9+8hOuHiS848fKDLMONWcAJ2LTNCssOrIeUn4/blI0aI1nUktB8IHr51996k4So4ARveguQ34fRR45irJ3QbacwQC+9ibn21rE62vmjSpAmbN2/mzJkzrFy5kkmTJtG6dWuuu+66Szrf5MmTmTRpkuNxXl4e0dF1N7W0ogn/f1cf4v2kFAVjIiIiIiIijZW1FPZ8YcwOO/Td2eOBLaHXeOh+N/gGV++cPk2NZZY9x0FuGmxfDNs+hoytxqyyvcvB3Rc63GKEZLHXg6t7Tb4raQyydkPS67Blwdllvn5hRojbczz4hZhbXx2p1hy44OBgXF1dyczMrHQ8MzOT8PDwi7+Iiwtt2rQhPj6ehx56iBEjRjBjxgwAx/Oqc05PT0/8/f0r3era6D5GEPfN7iw14RcREZEGozq7j5eWljJ9+nRiY2Px8vIiLi6O5cuXVxpjtVp5/PHHadWqFd7e3sTGxjr6zIqIOLXcdPj2GfhXF/hoTHkoZoF2N8NdC43G5Ff9pfqh2M8FREG/P8J9P8ADyXDNXyEoBkrzYdtH8P5IeLF9+c6Wa8Fmq4E3Jw2WzQb7VsC8ofBaAmx42wjFwrvB0Dfgz9vg2kcaTSgG1Zwx5uHhQc+ePVm5ciVDhgwBjOb7K1euZOLEiVU+j81mc/QIa9WqFeHh4axcuZL4+HjAmAGWlJTEhAkTqlNenWoT2oQ+MU1JPnyCj9en8qCa8IuIiIiTq9h9fPbs2SQkJDBr1iwGDBjAnj17LthDbsqUKbz33nvMmTOHDh068NVXXzF06FB++uknunfvDsBzzz3H66+/zjvvvEPnzp1Zv34948ePJyAggD/+8Y91/RZFRC6PzQaHVhmzw/Z8ebYpuW8I9BhjzPAKbFF7rx/SHm6YAtc/ZvSA2vqR0cssP9to8L/uTQhoAV2HQ9fbIaxT7dUizqUkH7Z8AGtnGztLAlhcjFmHV9wPLZx7Z8nLUe1dKRcsWMDYsWN544036NOnD7NmzeKjjz5i9+7dhIWFMWbMGCIjIx0zwmbMmEGvXr2IjY2luLiYL774gkcffZTXX3+d3//+94BxwfTss8/yzjvv0KpVKx5//HG2bt3Kzp07q7QbmFk7FS3ZlMZfFmwhMtCb7x+5Xk34RUQaCO1K2bBoV8qqq+7u4xERETz22GM88MADjmPDhw/H29ub9957D4Bbb72VsLAw/vvf/150zK/R5yQipis4YfQNWz8XThw8e7zlVUYz/Q63gZuHObVZy4yw7kI7W4Z2NvpEdRleu4Gd1F+5aZD8H2NmWEXfO09/I8jtc68x+7CBqpVdKQFGjRpFdnY2U6dOJSMjg/j4eJYvX+5onp+SkoLLObsU5Ofnc//995OWloa3tzcdOnTgvffeY9SoUY4xjzzyCPn5+fzhD3/g1KlTXHXVVSxfvrze/zIysEtzpi3bSfqpQn7Yl8116jUmIiKN3OHDh2nVqhWbNm1yzAQX53Apu48XFxefd73m7e3N6tWrHY+vvPJK/vOf/7B3717atWvHli1bWL16NTNnzrxoLWbvQC4iAoDdbszKWvdf2L4IrOX/X/L0h7g7oNc9ENrR3BoBXN2MhvxtEuHWfxkz2Sp2tszaAV/vKN/Zsm/5zpZDtbNlY5CabOwuuXPZ2ZmNQa3gigkQf2flXVEbuWrPGKuPzPwWcfqnO5n74yEGdA7jjbt71elri4hI7XDmGWPXXXcd8fHxzJo1q0bON27cOE6dOsXSpUurNL4+BmOaMVY1R48eJTIykp9++om+ffs6jj/yyCN89913JCUlnfecO++8ky1btrB06VJiY2NZuXIlgwcPxmq1OoItm83G3//+d55//nlcXV2xWq08/fTTlQK4n5s2bdoFdyDX5yQidaIk32h0v+6/RrP7CuHdoPfvoMsI8PQzr76qKjxphCLbPjZ2tqT8V38XN4i90Wja32GQdrZsSKylsPMTWPs6pK8/e7zVNcZyybb9wcXVvPrqWK3NGJPKRveJZu6Ph/h6VxaZeUWE+TvXL1AiIiIil+qll17i3nvvpUOHDlgsFmJjYxk/fjxz5851jPnoo4+YP38+77//Pp07d2bz5s38+c9/JiIigrFjx17wvGbvQC4ijVTWblj/X9jyIRSXz1R19YQuw4xd+iJ7OlcPJu8g6DnWuOWmG73Itn5khH37vjJu7j7n7Gx5g3a2dFYFJ4ylkslz4PRR45irp7GMNmEChHcxtbz6rlq7Usr52oY1oXdMEFabnY/Xp5pdjoiINGLjxo3ju+++46WXXsJisWCxWDh8+DDbt29n4MCB+Pn5ERYWxt13301OTo7jeQsXLqRr1654e3vTrFkzEhMTyc/PZ9q0abzzzjt88sknjvOtWrWq2nV999139OnTB09PT5o3b86jjz5KWVnZr74+wKpVq+jTpw++vr4EBgbSr18/jhw5ctl/VnK+S9l9PCQkhKVLl5Kfn8+RI0fYvXs3fn5+tG7d2jHmr3/9K48++ih33HEHXbt25e677+Yvf/mLox/thdSHHchFpJEoKzGWSb51i7FDX/J/jFCsaWvo/w94aDcMnQ1RvZwrFPu5gEi48sFzdrZ8pHxnywJjRtn7t8ML7eCzSXBkjXa2dBZZu+HTP8PMTrDySSMU8w2F6/4Of9kBg19VKFYFmjFWA0b3acG6wyf5IDmV+69rg4ua8IuINCx2u3HhaAZ3nypfiL/00kvs3buXLl26MH36dOPp7u706dOH3//+9/zrX/+isLCQv/3tb9x+++188803HDt2jNGjR/P8888zdOhQTp8+zQ8//IDdbufhhx9m165d5OXl8dZbbwHQtGnTapWfnp7OoEGDGDduHO+++y67d+/m3nvvxcvLi2nTpv3i65eVlTFkyBDuvfdePvjgA0pKSkhOTsbizL+Y1GOXs/u4l5cXkZGRlJaWsmjRIm6//XbHzwoKCir1nwVwdXXFpl+6RMRMp1KMGTYb50F+lnHM4gLtBxnLJVtdBy4NdB5JSHu44TG4/u9GD7VtHxvhYH62MWNu/X/P2dlyJIR1NrtiOZfNBge+MfqHHVh59nh4N2O5ZJdh4OZpXn1OSMFYDRjUtTnTlu0wmvDvz+HadiFmlyQiIjWptACeiTDntf9+tMq9PwICAvDw8MDHx8cxw+cf//gH3bt355lnnnGMmzt3LtHR0ezdu5czZ85QVlbGsGHDaNmyJQBdu3Z1jPX29qa4uPiiM4Z+zWuvvUZ0dDSvvPIKFouFDh06cPToUf72t78xdepUjh07dtHXP3HiBLm5udx6663ExsYC0LFjPWhy3IBNmjSJsWPH0qtXL8fu4/n5+YwfPx7gvN3Hk5KSSE9PJz4+nvT0dKZNm4bNZuORRx5xnPO2227j6aefpkWLFnTu3JlNmzYxc+ZM7rnnHlPeo4g0YjabESSs+6+xjNBeHtD7hRvLDXuMNWZWNRYWizETLqoX9H8aDn13dmfL3BRY/S/jFtrZaNrfdYR2tjRTSb6xzDdpNuTsLT9oMZbCXnE/tLzSuWc1mkjBWA3wcndlWI8o3v7pMB8kpSgYExGRemPLli18++23+Pmd3yT4wIED9O/fnxtvvJGuXbsyYMAA+vfvz4gRIwgKCqqR19+1axd9+/atNMurX79+nDlzhrS0NOLi4i76+k2bNmXcuHEMGDCAm266icTERG6//XaaN29eI7XJ+aq7+3hRURFTpkzh4MGD+Pn5MWjQIObNm0dgYKBjzMsvv8zjjz/O/fffT1ZWFhEREfzf//0fU6dOreu3JyKNVX4ObJoH69+CU+csx291jdE7rP0g9dZydYM2Nxq3W2fC3uWVd7ZcucNYqhd9hdG3yol2tiwqtZJ9upiMvCIycovIzDNuGXnFZOYV4eXuSoum3kQH+dCiqQ/R5bcA73ry70RumtE7bMPbUHTKOObRBHqMgT73QtNWZlbXIGhXyhqyN/M0/f/1Pa4uFtY8egOhasIvIuK0ztvF0EmWUsL5u1IOHDgQHx8fnnvuufPGNm/eHF9fX+x2Oz/99BP/+9//WLJkCRkZGSQlJdGqVavL3pVy2LBhBAQEOJZighHWxcfHc+TIEVq0aPGLrw+wadMmli9fzqeffsq2bdtYsWIFV1xxRZX/TLQrpfPT5yQi1Wa3Q2qSMTts51KwlhjHvQIg/i7odQ8EtzW1RKdQlZ0t2w80ZZdOm83OiYISMnKLyDpdREauEX5l5haRefpsCHayoPSSzu/v5UaLZj6OwCyqaXlwFuRNZJA3nm61vLtj6jpjueTOT8BuNY4FtYKE+yD+TvDS34e/RrtS1rF2YU3o1TKI9UdO8vGGNB64vo3ZJYmISE2xWJxmK3MPDw+sVqvjcY8ePVi0aBExMTG4uV34r32LxUK/fv3o168fU6dOpWXLlixZsoRJkyadd77q6tixI4sWLcJutztmjf344480adKEqKioX319gO7du9O9e3cmT55M3759ef/996sVjImISCNSfBq2LoB1c42ZThUiehi9wzoPAw8f8+pzNufubJl31OhFtu1jOLal8s6W7QdBt9trbGfLwhLrBWZ4lf8zt4jMvGKyThdRaq3aPB9PNxfCA7wIa+JFWIAX4f6ehPl7EervRVGJldSTBaScMG6pJwrJOVNMXlEZ29Pz2J6ed975LBYI9/cyZpc5Zpp5O2achfh5XlrvcWupEYStfR3S1589HnO1sVyy3QBwqeVArhFSMFaDRvdpwfojJ/kgOYUJ18aqCb+IiNS5mJgYkpKSOHz4MH5+fjzwwAPMmTOH0aNH88gjj9C0aVP279/Phx9+yJtvvsn69etZuXIl/fv3JzQ0lKSkJLKzsx29vGJiYvjqq6/Ys2cPzZo1IyAgAHf3ql/w3n///cyaNYsHH3yQiRMnsmfPHp544gkmTZqEi4sLSUlJF339Q4cO8Z///Iff/OY3REREsGfPHvbt28eYMWNq649PREScVeYOY3bY1gVQcsY45uZt9MXq/TuI6G5ufQ2Bf4Sxs+WVD0L2XiMg2/YxnDwE2xcaN++m0HkIdL0dohPO28DAarOTc6b4nICrIvA6eywjr4jTRWUXruFnLBZo5utJeIDnOaGXcQsL8CLM35Nwfy8CvN2rtXlPQUkZaScLSTle4AjNUk8UklZ+v6DEyrHcIo7lFpF86MR5z/d0cyEq6GxQ1qKpD1HnBGhNvH52LVVwwlgqmTzH2FkSwNXD+HO84j4I73rea0jNUTBWg27p1pwnP91B2slCVu/P4Rr1GhMRkTr28MMPM3bsWDp16kRhYSGHDh3ixx9/5G9/+xv9+/enuLiYli1bcvPNN+Pi4oK/vz/ff/89s2bNIi8vj5YtW/Liiy8ycOBAAO69915WrVpFr169OHPmDN9++y3XXXddleuJjIzkiy++4K9//StxcXE0bdqU3/3ud0yZMgXgF18/MzOT3bt3884773D8+HGaN2/OAw88wP/93//Vxh+diJjBZoW9X4FPM+MXP83kkeooKzZm16z7L6SuPXu8WVsjDIu7w5jxJDUvpN05O1tuhG0fYdu+GJf8LFg/F9bP5bRnOJsDEvnG41o2FkeSmVtE9plirLaqzfLy8XA1Ai5/L8IDvAgtD7nOhl5ehDbxxN215ncP9fFwo11YE9qFNTnvZ3a7nRP5JUZYdrKQ1BMFpJ44O+PsWG4RxWU2DmTncyA7/4LnD/JxJ7qpD718s7m1cBndcr7AzVZknN83FEvv30Ov8eAXWuPvTc6nHmM1bNqyHbz902EGdgnn9d/2NLUWERG5NL/Uk0qcj3qMOT99Tg2UzQpL7oNtHxmPLa4Q2hEi4o1lb5E9jN3w3DxMLVPqoROHYMNbsOk9KDhuHHNxgw63GoFYzNXana8GlVptZJ02ZnRlls/oysgrIiuvuNJSx6KSEvq67GSwy4/c7LqOJpZCxzl226JZZr2SZbYrOUoIoU2M2VwVoVdYRQDm72XM/vL3ws/TrVqzvOqLUquNY6eKzplpVlApRDuRX8w1Llu5x3U517lucTxvh60l/y0byBf2vjQL8D+7NDPIhxbNzs44C/bzcMo/FzNU9fpBwVgN25NxmgGzvsfNxcJPk28gtIl+oRIRcTYKxhoWBWPOT59TA2SzwicPwJYPjEDDpxmcyTx/nKsHhHUxlsFF9jD+GdJBPXYaI5vV2CFx3ZuwfyWOJvD+kdBznLFDX5NwMyt0Ona7ndzCUkcvr6y8YkfodbaBfTHH84upamrQxMuNcH8voppYuM6yib4F3xB78kdc7Wcb4Nujr8DSdQR0Hgq+wbX07uqhkgLY+iG2Na/jcnwvAHYs7A64hmXeg/k6P5bUU4UUldp+8TTe7q5El++iGX3OUs2KY76eWhhYQc33TdI+vAk9Wwax4chJPl6vJvwiItKwPPPMMzzzzDMX/NnVV1/Nl19+WccViYjTsdlg2R+NUMziCiPmQqfBRmPvo5uMZVlHNxr3C0+W398I6/9rPN/dB5rHGSFZRHlY1rT1eb2MpIE4nQmb3oUN70Bu6tnjsTcas8PaDgBX/Vr7c0WlVrJPF1+ggX2xY9ZXZp6x5K8q3Fws5bO6PI1ljU2MmV4VSx0rjvt4nPtZXAP8yfjveNenRj+yQz9gSV1rLH1d/qjRrL/rSKN5vwk7W9aJ3HRYNwfWvwVFp3AB8GgCPcZg6XMvHZu2oiPwN4ywMvtMMaknKi/RTD1p9Dg7lltIYamVvZln2Jt55oIv18zXo9IOmuf2OWse4IVbLSw9dXaaMVYLFm5I4+GPtxDd1JvvHr5eTfhFRJyMZoxd3IkTJzhx4vwmswDe3t5ERkbWcUW/TjPGnJ8+pwbEZoPP/gQb3zVCseFvQpdhFx5rt8PJw0ZAdnQjHN1s3EpOnz/WM6B8CeY5M8sCorWczlnZ7XDkR2N22K5PwVbeiN07CLr/FnqOh2ax5tZoEpvNzomCEmOGV/mMrsozvIzA62RB6a+frFyQj3ulpYznNq2vWOrY1MejZn6vzTsK2xeX72y5+ezxip0tu440wrKGsIQ6bT2sfQ12LAV7+Q7fQTGQMAHi7wSv6v99VlJm4+ipQkdYlnKigLQTZx+f+pXP3dXFQkSg1zk7aZbfygO0pr4Na5mmllKaqLDESp9nvuZ0URnzfteHq9uqCb+IiDNRMNawKBhzfvqcGgi7HT6fZDTmtrjAsDnGjoHVYbPB8X3nzCzbBBlboazo/LE+wWdDsoqZZU3Caua9SO0oyoUtHxr/jmTvPns8qo8xO6zTEHBvuH8vF5SUkXlO366KWV1nd3AsJut0EaXWqv0K7+HmUh5ueZ7Tv6tyP69Qf0+83E1ampy919jJcutHxs6WFbyDjGWWXUdC9BXONRvUWgq7lsHa1yFt3dnjMVfDFROg3c21uhQ8r6jUMdMs9cTPArSThZT8ygxBHw/X83bQdARoQT54ezjXMnYFYyZ74pPtvLPmCIO6hvPaXWrCLyLiTBSMNSwKxpyfPqcGwG6HLx42ZgBhgaFvQNyomjm3tRSydp0zs2wTZO44O8voXP6R5UFZ+cyy5vHg07Rm6pBLd3SzsVR220IoLTCOuftCt5HQ63fQvJup5dUUu91OZl4xB7PPcCD7DAey8zmUk8/RU4Vk5BVxuugC/85eRLCfxznhVuWm9RXHA33cnWP2j91evrPlx7B9EeRnnf1ZQDR0GW6EZGGd6+8s0IITsPEdSJ4DeenGMVcPo+6E++rFv8M2m52s08VGUHb8/BlnmaeLfrWXXLCfJy2aep/ta+boc+ZN8wBvXOvZajkFYybbnZHHzbN+wM3FwprJNxLSxNPskkREpIoqgpSYmBi8vb3NLkcuU2FhIYcPH1Yw5sT0OTk5u93oJZQ0G7DAkNchfnTtvmZpkRGOHd14dmZZ9m4cDdvPFdTqnCWYPYxfYD2b1G59AqWFsGMJrPsvpK8/ezykozE7rNvt4BVgXn2XobDEysGcMxzMzudgdj4Hss9wMOcMh7LzyS+x/uJzvd1dy2d1eZ5d1tjk7Eyv8AAvQvw88XBzollU1WEtg8PfGyHpzmWVl06HdDTC0i4jIKileTWeK3uP8f+2zR9AWfkunL4h0Pv30Ose8As1t75qKC6zkn6ysNIOmucGaL8W3Lq7WogI9L7wjLMgH1OCWgVj9cDQ135kU8op/nZzByZc1zjXwIuIOCOr1crevXsJDQ2lWbNmZpcjl+n48eNkZWXRrl07XF0rLwGor9cQUpk+Jydmt8NXj8HaV43Hg181ekSZofiMsezy3Ob+Jw5eYKAFQtqfXYIZ2cPYGbMBL+GrU8cPGEslN883mrIDuLgbGzD0/h206Ft/ZwWdw263cyy36GzwlX2GgzlGEJZ+qvCiz3N1sRAd5E1siB+tQ3xpHeJHVJC3IwRr4unmHLO86kJpIez9yphJtu9/YC05+7PoBGM2lhk7W9rtcGClsVxy/9dnj4d3hSvuN2a4uTW8iTG5BaWVlmZWbAyQdrKQtJMFv7rEt4mnW/mmAGd31KwIz1oF+9XKbDMFY/XAx+tT+evCrbRo6sOqh69TE34RESdy7NgxTp06RWhoKD4+PrpIdUJ2u52CggKysrIIDAykefPm542pr9cQUpk+Jydlt8OKx+Gnl43Ht/0beo41t6afKzxZ3tS/YmbZZshLO3+cixuEdqo8syy0I7i613XFzslaBnu/NGaHHfz27PGAFtBrHHS/u97OrMkvLuNQTkX4lc/BnHwOZJ3hUE4+haUXn/0V4O1ObHnw1TrEl9gQP2JDfGnR1LfhzvaqTYWnyne2/AgO/YBj9qfF9ezOlh1uqd2dLUsKYOuHsHY25OwpP2gxXveKCdCyn1OEurXBarOTmVfkCMxSy2edVTzOOl38i8/f8eQAfD1rfndZBWP1wLlN+N/7XQJXta3jJFtERC6Z3W4nIyODU6dOmV2KXKbAwEDCw8MvGG7W12sIqUyfkxOy22Hlk7D6X8bjW/9lLCtyBmeyKjf3P7oR8rPPH+fmZcwQOXdmWbM2tdpY2+nkHTP6Lm14B04fLT9ogbb9jdlhbRLrxZ+XzWbnaG5h+dJHo/dXxVLIY7kX2NihnJuLhRZNfWhdHnpVzABrHezb4Hb3q1fyjsGOxUbT/nN3tnTzhg4VO1veWHM7W+amw7o5sOHts7McPZpAj7uhzx+gaauaeZ0GrKjUSppjpllhpRlnBSVWvn/k+lp5XQVj9cTUT7bz7poj3NK1Oa/e1cPsckREpJqsViulpVXf8lzqF3d39/OWT56rPl9DyFn6nJyM3Q7f/AN+eMF4POgF6HOvuTVdDrsdctMqN/c/usnYQfHnPPyMhv4R8Wd3xAxq1bhmkdjtcOg7Y3bY7s/BXj6ryifYCBJ6joOgGFNKO1NcxqFzlj4eKF/6eCjnDEWlF9+tr6mvB62DzwZfFcsgWzT1wd1Vs79MlbPP6Ee27aPKS6O9g4xdTLuONJbnXsrOlmnrYe1rsGPp2X+Pg2KMZvrxd4GX/j6q7xSM1RM7j+Yx6N9qwi8iIlIf1edrCDlLn5OT+XYGfPescf/m5+CK+8ytpzbY7cYv4efOLDu2BUrzzx/rFVh5CWZEd/CPaHhhWeFJ2Py+0T/s+P6zx1tcacwO63hbnfRdstrsHD1VeM7SxzMcyDL+mZl38eVc7q7G7C8j9KpY/uhL62A/gnxraOaR1B673Qiuty00drY8k3n2Z/5R0LViZ8suv/zfnrUUdi0z+oelrTt7POZqY7lku5vrxSxHqRoFY/XIkFd/ZHPqKR4d2IH7rlUTfhERkfqivl9DiEGfkxP57nn49mnj/oBnoO8D5tZTl2xWY4e6c2eWZWyr3DC8gl9Y5SWYEd3rvoF4TUnfYMwO274IysqXHXo0gbhR0Ot3ENapVl72dFHpOY3vzy59PJiTT0nZxWd/Bft50Dq4ovG9ryMIiw7yxk2zvxoGmxUOle9suWsZFOed/VlIByMg6zqi8szFghPGst/kOZCXbhxz9TDGJtxn7FYrTkfBWD3y0fpUHlm4lZhmPnzzkJrwi4iI1Bf1/RpCDPqcnMQPL8LK6cb9m56Cfn80t576oKwEsnZWbu6ftfPssqxzBbQ4ZwlmD+O+V0AdF1xFJQWwfaERiJ3b4ymsK/S+xwgTPJtc9stYbXbSThacDcDKG98fzMkn+xeaeXu4utCymU+lnR9bh/gSG+xHgI82TGhUSguNHS23fWzscPnznS07DzMa6W/+AMrKdxP1DYHevzf6ItbTTSGkahSM1SMFJWUkPL2S08VlzP99Av3aOOm3QSIiIg1Mfb+GEIM+JyewehZ8/YRx/8Yn4OpJppZTr5UUQOb28qCsfGZZzj4cu+ydq1mbyjPLwruCh2+dl+yQvRfW/9cIEYrLe6y5ekDnoUaQENX7kpaI5haWnm16f84MsMM5BZRYLz77K6SJZ3nvL6P5fUUQFhXkg6smI8jPOXa2/NiYUfbz/+bCukLf+6HL8DpZ9iu1r6rXDzW/H6acx8fDjSHdI5m39gjvJ6coGBMRERGRhuOnV86GYjdMUSj2azx8ILqPcatQlGf0KHPMLNsEp44YvbqO7zd+kQewuEBIx/KeZd2Nf4Z1qd1f4q2lsPszY3bY4R/OHg+KMWbUxP8WfJv96mnKrDZSTxZWCr4qen/lnLnActNyHm4utGrmS2yo7zlLII1/+ntp9pdUg3egsQFEj7vP7my5+3PwaWbsLhlzVcPr/SdVohljdaSiCb+7q9GEP9hPCbSIiIjZnOEaQvQ51WtrXoOvJhv3r5sM1z1qbj0NSf5xOLYJ0jed7Vt2+tj541zcIbzL2ZllEd2NPkqulzkHIjcNNrwNG98928jc4gLtBhrLJVvfcMGd/k4VlDhmfjlmgOXkc+R4PqXWi//qGebvWSn4qpgBFhHordlfInJJNGOsnukU4U9cdCBbUk+xaEMa/6cm/CIiIiLizJL+czYUu+YRhWI1zbcZtEk0bhXyjlVu7p++EQpPlB/bBMw1xrn7QHi3c3bD7A5NYy8YZFVis8HBb2DdXNj7JdjLlzH6hUGPMdBzHAREUWq1kXq84Lyljwey8zmRf/HZX55uLrQKNgKv2HNmfrUK9qWJZn+JiEkUjNWhO/tEsyX1FB8kp/CHa1pj0TRNEREREXFG696EL/9q3L9qElz/d3PraSz8mxu3DoOMx3Y7nEqpvATz6GYoOQ2pa41bBc8AiIirPLMssIWxdCz/OGx+D9a/BScPOZ5S2uIqUlrfwUbvfuw/UcyBpRkczNlPyvECymwXn/3VPMDLmPn1sxlgEQHe2ohMROodBWN16La4CJ76bBeHjxew5sBxrlSvMRERERFxNuvfgs8fMu73+xPcOFV9ecxisUBQS+PWeahxzGaDEwcqN/c/ttVoln/o+/Km4+V8mmELbgfpG3GxGrs8Frr48Y3njfy36Do27g2DvQC7zntpb3dXWgX7Grs9Vuz6GOJHq2BffD31a6aIOA/9H6sOGU34I3hvbQrvJ6coGBMRERER57LxXfjsz8b9vhMh8UmFYvWNiwsEtzVucaOMY9YyyN7tCMrs6RuxZ+7ApeA4LilrANhqa8V71kQ+tfalsMDLcbrIQO/y2V9nlz7GhvgR7u+l2V8i0iAoGKtjo/u04L21KXy1I4PjZ4pppib8IiIiIuIMNs2HZX807idMgP7/UCjmLFzdILwLR9xbsSgngUUn08kuyKODJYX2LqkccW1JQUgcrYP9uC+kYvmj0fvLx0O/MopIw6b/y9WxzhEBxEUFsCUtl0Ub0/jDNWrCLyIiIiL13JYP4ZMHADv0+QPcPEOhmJM4U1zGF9uOsXBDGsmHTjiON/Hypkvc9YzoGUX36ED1PxaRRkvBmAlG92nBlrRtfJCcyr1Xqwm/iIiIiNRjWz+GpRMAO/T6HQx8XqFYPWez2Uk6dIKFG9L4cvsxCkqsgPGxXdUmmJG9ounfKQwvd1eTKxURMZ+CMRPcFhfBPz7fxaGcfNYePEHf2GZmlyQiIiIicr7ti2DJH8Bug57jYNALCsXqsdQTBSzamMaijWmknih0HG8d7MvwnlEM6xFJ8wBvEysUEal/FIyZwNfTjcHxEcxPMprwKxgTERERkXpnx1JYdK8RinW/G275l9HYXeqVgpIyvtyWwccbUll78OxSST9PN26La86InlH0aBGkVSoiIhehYMwko/u0YH5SCl9tVxN+EREREalndn0Ki34HdivE3wW3/VuhWD1it9tJLl8q+cW2Y+Sfs1SyX2wwI3pGMaBzON4eWiopIvJrFIyZpEtkAN2iAtialsvijence01rs0sSEREREYHdn8PH48BWBt3ugN+8rFCsnkg7WcDijeks3JBGyokCx/GWzXwY0SOKYT2jiAzUUkkRkepQMGai0X1asDVtGx8kp/D7q1tperOIiIiImGvPcvhorBGKdR0JQ14DF806MlNhiZXlO47x8fo0fjpw3HHc18OVW7tFMKJXFL1aaqmkiMil0lc/JrotLgJfD1cO5uSTdM7WySIiIiJmefXVV4mJicHLy4uEhASSk5MvOra0tJTp06cTGxuLl5cXcXFxLF++vNKYmJgYLBbLebcHHnigtt+KVNfe/8FHd4OtFDoPgyGzFYqZxG63s/7wCR5dtJXeT3/NXxZscYRiV8Y2Y+btcaybkshzI7rRO6apQjERkcugGWMm8vN04zfxkXyQnMIHySlc0VpN+EVERMQ8CxYsYNKkScyePZuEhARmzZrFgAED2LNnD6GhoeeNnzJlCu+99x5z5syhQ4cOfPXVVwwdOpSffvqJ7t27A7Bu3TqsVqvjOdu3b+emm25i5MiRdfa+pAr2fw0LfgvWEug0GIbNAVf9qlDXjp4qZPHGNBZuSOPw8bNLJaObejOiRzTDekQS3dTHxApFRBoei91ut5tdxOXKy8sjICCA3Nxc/P39zS6nWran53Lry6vxcHVh7d9vpKmvh9kliYiINBrOfA1RGxISEujduzevvPIKADabjejoaB588EEeffTR88ZHRETw2GOPVZr9NXz4cLy9vXnvvfcu+Bp//vOf+eyzz9i3b1+VZ7noc6plB76FD+6AsiLocCuMfBtc3c2uqtEoKrXy1Y4MFm5IY/X+HCp+O/PxcGVQ1+aM7BlF75imuLhoVpiISHVU9fpBXwOZrEtkAF0jA9iWnsvijWn8/mo14RcREZG6V1JSwoYNG5g8ebLjmIuLC4mJiaxZs+aCzykuLsbLy6vSMW9vb1avXn3R13jvvfeYNGnSL4ZixcXFFBcXOx7n5eVV561IdRz87mwo1v4WGPGWQrE6YLfb2ZhyioUb0vhsy1FOF5c5fpbQqikje0UzsEs4vp76dU1EpLbp/7T1wOg+Ldi2ZBvvJ6fwu6vUhF9ERETqXk5ODlarlbCwsErHw8LC2L179wWfM2DAAGbOnMk111xDbGwsK1euZPHixZWWTp5r6dKlnDp1inHjxv1iLTNmzODJJ5+8pPch1XB4Nbw/ygjF2t1szBRz0+qF2pSRW8TiTcZSyYPZ+Y7jkYHejOgZxfAeUbRopqWSIiJ1ScFYPfCb+Aj+8flODmbnk3zoBAnqNSYiIiJO4KWXXuLee++lQ4cOWCwWYmNjGT9+PHPnzr3g+P/+978MHDiQiIiIXzzv5MmTmTRpkuNxXl4e0dHRNVp7o3fkJ5h/O5QVQpub4PZ3FYrVkqJSKyt2ZvLxhjRW78vGVr5U0tvdlYFdwxnRM4orWjXTUkkREZMoGKsH/DzdGBwfwQfJqXyQnKJgTEREROpccHAwrq6uZGZmVjqemZlJeHj4BZ8TEhLC0qVLKSoq4vjx40RERPDoo4/SuvX5rSGOHDnC119/zeLFi3+1Fk9PTzw9PS/tjcivS1kL80dCaT7E3gCj3gM3/XnXJLvdzpa0XD5en8qnW46SV3R2qWSfmKaM6BnFoG7N8dNSSRER0+n/xPXE6D4t+CA5lS+2Z/BEfglBasIvIiIidcjDw4OePXuycuVKhgwZAhjN91euXMnEiRN/8bleXl5ERkZSWlrKokWLuP32288b89ZbbxEaGsott9xSG+VLVaWug/dGQMkZaH0d3PE+uHv96tOkarLyili8KZ2FG9LYn3XGcTwiwIvh5UslY4J9TaxQRER+TsFYPdEtKpAukf5sT89jkZrwi4iIiAkmTZrE2LFj6dWrF3369GHWrFnk5+czfvx4AMaMGUNkZCQzZswAICkpifT0dOLj40lPT2fatGnYbDYeeeSRSue12Wy89dZbjB07Fjc3XX6aJm0DvDcMSk5DzNVwxwfg7m12VU6vuMzK1zuzWLghle/2nl0q6eXuws2dwxnZK5q+rbVUUkSkvtKVST0yuk8LHluynQ/UhF9ERERMMGrUKLKzs5k6dSoZGRnEx8ezfPlyR0P+lJQUXFxcHOOLioqYMmUKBw8exM/Pj0GDBjFv3jwCAwMrnffrr78mJSWFe+65py7fjpwrfSPMGwrFedDyKrhzAXioyfulstvtbEvPZeGGND7ZfJTcwlLHz3q2DGJk+VJJfy/t8CkiUt9Z7Ha73ewiLldeXh4BAQHk5ubi7+9vdjmX7HRRKQnPrKSgxMpH/9eXPq2aml2SiIhIg9ZQriEaOn1Ol+noZnj3N1CUCy36wl0LwdPP7KqcUtbpIj7ZdJSFG9LYk3nacbx5gBfDekQyvEcUrUP0ZysiUh9U9fpBM8bqkSZe7vwmLoIP1xlN+BWMiYiIiMhlydgG84YYoVh0Atz1sUKxaiops7FyVyYLN6Sxam821vK1kp5uLgzobOwq2a9NMK5aKiki4pQUjNUzo/u04MN1qXy+7RhP3NaJQB814RcRERGRS5C5A975DRSehMhe5TPFmphdlVOw2+3sOJpXvlQynZMFZ5dKdm8RyIieUdzaLYIAby2VFBFxdgrG6pluUQF0au7PzmN5LNqYzu+uamV2SSIiIiLibLJ2lYdiJyCiB9y9GLy0DPXX5JwpZmn5rpK7M84ulQxt4smwHlGM6BlFm1DNuBMRaUgUjNUzFouFOxNaMGWp0YT/nn4xasIvIiIiIlWXvQfeuQ0KcqB5PNy9BLwCzK6q3iops/HtniwWbkjj291ZlJUvlfRwc6F/pzBG9IziqjbBuLm6/MqZRETEGSkYq4cGx0fw9Oe72J91hvVHTtI7Rr3GRERERKQKcvYZoVh+NoR3M0Ix70Czq6qXdpYvlVy6OZ0T+SWO43HRxlLJ33SLIMBHSyVFRBo6BWP1UEUT/gXrU/kgKUXBmIiIiIj8uuMH4O1b4UwmhHWFMZ+Aj64jz3Uiv8SxVHLnsTzH8ZAmngzrHsnwnlG0C1MfNhGRxuSS5gO/+uqrxMTE4OXlRUJCAsnJyRcdO2fOHK6++mqCgoIICgoiMTHxvPHjxo3DYrFUut18882XUlqDMTqhBQCfbTvGqYKSXxktIiIiIo2aIxTLgNDOCsXOUWq1sWJnJv83bz0Jz3zN9M92svNYHh6uLgzqGs5b43qz5tEbmDyoo0IxEZFGqNozxhYsWMCkSZOYPXs2CQkJzJo1iwEDBrBnzx5CQ0PPG79q1SpGjx7NlVdeiZeXF8899xz9+/dnx44dREZGOsbdfPPNvPXWW47Hnp6el/iWGoa4qAA6Nvdn17E8lmxKZ3w/NeEXERERkQs4cchYPnn6KIR0MEIx32ZmV2W63Rl5LFxvLJXMOXP2i+aukQGM7BXFbd0iCPLVDvAiIo2dxW6326vzhISEBHr37s0rr7wCgM1mIzo6mgcffJBHH330V59vtVoJCgrilVdeYcyYMYAxY+zUqVMsXbq0+u8AyMvLIyAggNzcXPz9G85uO/PWHObxT3bQNtSP//3lGjXhFxERqWEN9RqiodHn9AtOHoG3b4HcVAhuB+M+B7/zv6xuLE7ml7Bsy1EWbkhjW3qu43iwnwdDy5dKdgjXv0MiIo1BVa8fqjVjrKSkhA0bNjB58mTHMRcXFxITE1mzZk2VzlFQUEBpaSlNm1ae2r1q1SpCQ0MJCgrihhtu4B//+AfNmjXub7oGd4/kmS92sy/rDBuOnKSXeo2JiIiISIVTKfDOrUYo1qwNjP20UYZiZVYb3+/L5uP1aXy9K5NSq/G9v7urhRs7GLtKXts+BHftKikiIhdQrWAsJycHq9VKWFhYpeNhYWHs3r27Suf429/+RkREBImJiY5jN998M8OGDaNVq1YcOHCAv//97wwcOJA1a9bg6up63jmKi4spLi52PM7LyztvTEPg7+XObXHN+Wh9Gu8npygYExERERFDbpqxfPJUCjSNhbGfQZNws6uqU3szT7NwQxpLNqWTffrs7wadI/wZ0TOKwfGRNNVSSRER+RV1uivls88+y4cffsiqVavw8vJyHL/jjjsc97t27Uq3bt2IjY1l1apV3HjjjeedZ8aMGTz55JN1UrPZRvdpwUfr0/h86zGeuLWztowWERERaezyjhqN9k8ehqBWxkwx/+ZmV1UncgtKWbbF2FVyS9rZpZJNfT0YEh/JiJ5RdIrQUkkREam6agVjwcHBuLq6kpmZWel4ZmYm4eG//A3VCy+8wLPPPsvXX39Nt27dfnFs69atCQ4OZv/+/RcMxiZPnsykSZMcj/Py8oiOjq7GO3Ee8dGBdAhvwu6M0yzZlMY4NeEXERERabxOZ5SHYocgsCWM+wwCIn/9eU6szGrjh/05LNyQxoodmZRYbQC4uVi4vkMoI3tGcV37UDzctFRSRESqr1rBmIeHBz179mTlypUMGTIEMJrvr1y5kokTJ170ec8//zxPP/00X331Fb169frV10lLS+P48eM0b37hb748PT0bza6VFouFOxNaMPWTHXyQnMrYK2PUhF9ERESkMTqdaYRiJw5AQIvyUCzK7Kpqzf6sM+VLJdPIzDu7VLJDeBNG9opmcHwEwX6N43cCERGpPdVeSjlp0iTGjh1Lr1696NOnD7NmzSI/P5/x48cDMGbMGCIjI5kxYwYAzz33HFOnTuX9998nJiaGjIwMAPz8/PDz8+PMmTM8+eSTDB8+nPDwcA4cOMAjjzxCmzZtGDBgQA2+Vec1OD6SZ77YxZ7M02xMOUnPluo1JiIiItKonMk2eood3wf+UTDuUwhsYXZVNS63sJTPthq7Sm5KOeU4HuTjzuDypZKdI/z1RbGIiNSYagdjo0aNIjs7m6lTp5KRkUF8fDzLly93NORPSUnBxeXsNObXX3+dkpISRowYUek8TzzxBNOmTcPV1ZWtW7fyzjvvcOrUKSIiIujfvz9PPfVUo5kV9msCvN25rVsEH29I4/2kVAVjIiIiIo1Jfo4RiuXsAf9IIxQLijG7qhpjtdlZXb5U8qsdGZSUGUslXV0sXN8+hBE9o7ihQ5iWSoqISK2w2O12u9lFXK68vDwCAgLIzc3F379hNtvcmHKSYa/9hKebC8l/T1QTfhERkRrQGK4hGoJG/TnlHzdCsawd0KQ5jPscmsWaXVWNST1RwF1vJpFyosBxrF2YHyN7RjOkeyQhTfRFuYiIXJqqXj/U6a6Ucum6n9OEf+nmdMZeGWN2SSIiIiJSmwpOwLzBRijmFwZjP2tQoRjAnB8OknKigABvdwbHRzCyZzRdIrVUUkRE6o7mIzsJi8XC6D5GH4kPklNoABP9RERERORiCk/CvCGQsQ18Q41QLLiN2VXVqJIyG59uOQrAv0d3Z/rgLnSNClAoJiIidUrBmBMZ0j0SL3cXdmecZlPqKbPLEREREZHaUHgK5g2FY1vAJxjGfgoh7cyuqsZ9vzebkwWlBPt50i+2mdnliIhII6VgzIkEeLtza7cIAN5PSjG5GhERERGpcUW58N4wOLoJfJoZoVhoB7OrqhVLNqUDMDg+AjdX/VoiIiLm0N9ATqZiOeVnW4+SW1hqcjUiIiIiUmOK8uC94ZC+AbyDYMwyCOtkdlW1IrewlBW7MgEY2j3S5GpERKQxUzDmZHq0CKR9WBOKSm18sjnd7HJEREREpCYUn4b5IyFtHXgFGqFYeBezq6o1y7cfo6TMRttQPzpHNLKdRkVEpF5RMOZkjCb80YCxnFJN+EVEREScXEk+zL8dUteCVwCM+QSadzO7qlq1eKPxBe/QHpFqti8iIqZSMOaEhnaPwtPNaMK/WU34RURERJxXSQG8PwpSfgLPALh7CUTEm11VrUo/VUjSoRMADI7XMkoRETGXgjEnFODjzi3dmgNqwi8iIiLitEoK4INRcPgH8GgCdy+GyJ5mV1XrlpY33b+idVMiA71NrkZERBo7BWNO6q4Eown/p1uPklekJvwiIiIiTqW0ED68Ew59Dx5+8NtFENXL7Kpqnd1ud+xGOax7lMnViIiIKBhzWj1aBNEuzM9owr9JTfhFREREnEZpESz4LRz8Ftx94a6F0CLB7KrqxI6jeezPOoOnmws3dw03uxwREREFY87KaMJvzBqbryb8IiIiIs6hrBg+uhv2fw3uPnDXx9Cyr9lV1ZmKpvuJncLw93I3uRoREREFY05taPdIRxP+LWm5ZpcjIiIiIr+krAQ+Ggv7/gdu3nDnRxDTz+yq6kyZ1cayLUcBGKqm+yIiUk8oGHNigT4e3NLVaML/gZrwi4iIiNRf1lJYOB72fgluXnDnh9DqarOrqlOr9+eQc6aYIB93rm0fYnY5IiIigIIxpze6vAn/si1qwi8iIiKX79VXXyUmJgYvLy8SEhJITk6+6NjS0lKmT59ObGwsXl5exMXFsXz58vPGpaen89vf/pZmzZrh7e1N165dWb9+fW2+jfqlIhTb/Rm4esId70Pr68yuqs5VNN2/LS4Cd1f9GiIiIvWD/kZycr1aBtE21I/CUiufbD5qdjkiIiLixBYsWMCkSZN44okn2LhxI3FxcQwYMICsrKwLjp8yZQpvvPEGL7/8Mjt37uS+++5j6NChbNq0yTHm5MmT9OvXD3d3d7788kt27tzJiy++SFBQUF29LXNZy2DR72HXp+DqYYRibW40u6o6d6a4jK92ZABGOxAREZH6QsGYkzu3Cf/7asIvIiIil2HmzJnce++9jB8/nk6dOjF79mx8fHyYO3fuBcfPmzePv//97wwaNIjWrVszYcIEBg0axIsvvugY89xzzxEdHc1bb71Fnz59aNWqFf379yc2Nrau3pZ5rGWw5A+wcym4uMOo96BtotlVmeKr7RkUldpoFexLfHSg2eWIiIg4KBhrAIb1iMTDzYVdx/LYqib8IiIicglKSkrYsGEDiYlngxsXFxcSExNZs2bNBZ9TXFyMl5dXpWPe3t6sXr3a8XjZsmX06tWLkSNHEhoaSvfu3ZkzZ07tvIn6xGaFpRNg+6LyUGwetBtgdlWmWbrZWEY5JD4Si8VicjUiIiJnKRhrACo14U9WE34RERGpvpycHKxWK2FhYZWOh4WFkZGRccHnDBgwgJkzZ7Jv3z5sNhsrVqxg8eLFHDt2zDHm4MGDvP7667Rt25avvvqKCRMm8Mc//pF33nnnorUUFxeTl5dX6eZUbFb45AHY9hG4uMHIt6H9QLOrMk1mXhE/7s8BtIxSRETqHwVjDUTFcsplW45yWk34RUREpA689NJLtG3blg4dOuDh4cHEiRMZP348Li5nLzFtNhs9evTgmWeeoXv37vzhD3/g3nvvZfbs2Rc974wZMwgICHDcoqOj6+Lt1AybDZb9EbZ8ABZXGDEXOt5qdlWm+mRzOjY79GwZRItmPmaXIyIiUomCsQaid0wQbUL9KChRE34RERGpvuDgYFxdXcnMzKx0PDMzk/Dw8As+JyQkhKVLl5Kfn8+RI0fYvXs3fn5+tG7d2jGmefPmdOrUqdLzOnbsSErKxWe5T548mdzcXMctNTX1Mt5ZHbLZ4LM/web3jFBs+JvQabDZVZluySbj2lSzxUREpD5SMNZAqAm/iIiIXA4PDw969uzJypUrHcdsNhsrV66kb9++v/hcLy8vIiMjKSsrY9GiRQwefDYM6tevH3v27Kk0fu/evbRs2fKi5/P09MTf37/Srd6z2+HzSbDxXbC4wLD/QJdhZldlut0Zeew6loe7q4VbuzU3uxwREZHzKBhrQIZ1N5rw7zyWx7Z0NeEXERGR6pk0aRJz5szhnXfeYdeuXUyYMIH8/HzGjx8PwJgxY5g8ebJjfFJSEosXL+bgwYP88MMP3HzzzdhsNh555BHHmL/85S+sXbuWZ555hv379/P+++/zn//8hwceeKDO31+tsdvhi4dhw1uABYbMhq4jzK6qXliyyWi6f337UAJ9PEyuRkRE5HxuZhcgNSfI14NBXcJZuvkoHySn0C0q0OySRERExImMGjWK7Oxspk6dSkZGBvHx8SxfvtzRkD8lJaVS/7CioiKmTJnCwYMH8fPzY9CgQcybN4/AwEDHmN69e7NkyRImT57M9OnTadWqFbNmzeKuu+6q67dXO+x2WP4orHsTIxR7HeJGmV1VvWC12fmkfBnlsB5aRikiIvWTxd4A1tzl5eUREBBAbm6uc0y1r0VJB48z6j9r8fFwJfmxRPw8lX2KiIhcjK4hnEO9/ZzsdvjqMVj7qvF48KvQ/bfm1lSP/LQ/hzvfTMLfy411UxLxdHM1uyQREWlEqnr9oKWUDUyfVk1pHeJLQYmVZWrCLyIiIlI77HZY8fjZUOy2fysU+5nF5csob+kWoVBMRETqLQVjDYzFYuHOiib8yUdMrkZERESkAbLb4etp8NPLxuNb/wU9x5paUn1TWGLly23HAC2jFBGR+k3BWAM0vEcUHq4ubE/PY1uamvCLiIiI1Bi7Hb75B/w4y3g86AXodY+pJdVHK3Zlkl9iJSrIm54tgswuR0RE5KIUjDVAQb4eDOwaDsD7ySkmVyMiIiLSgKx6Fn54wbh/83PQ515z66mnlmxMA2Bo90hcXCwmVyMiInJxCsYaqNHlyymXbU7nTHGZydWIiIiINADfPQ/fPWvcH/AMXHGfufXUUzlnivl+Xw4AQ7prGaWIiNRvCsYaqIRWTWkd7Et+iZVPt6gJv4iIiMhl+f4F+PZp4/5NT0HfB8ytpx77dMtRrDY7cVEBxIb4mV2OiIjIL1Iw1kBZLBbHrLEPtJxSRERE5NKtngXfPGXcv/EJ6PdHU8up75aU70Y5VLPFRETECSgYa8CG9zSa8G9Ny2V7uprwi4iIiFTbTy/D108Y92+YAldPMreeem5/1hm2puXi6mLh1rgIs8sRERH5VQrGGrCmvh7c3EVN+EVEREQuyZrX4H9TjPvXTYZr/mpuPU5gaflssWvbhRDs52lyNSIiIr9OwVgDV7Gc8pNN6eSrCb+IiIhI1ST9B76abNy/5hG47lFz63ECNpudpZuNYExN90VExFkoGGvgrmjdlFZqwi8iIiJSdevehC/LZ4dd/RBc/3dz63ES64+cJO1kIX6ebvTvFGZ2OSIiIlWiYKyBM5rwRwNqwi8iIiLyq9a/BZ8/ZNzv9ye44XGwWMytyUks2ZQGwMAu4Xi5u5pcjYiISNUoGGsEhveIwt3VwhY14RcRERG5uI3vwmd/Nu73nQiJTyoUq6KiUiufbT0GaDdKERFxLgrGGoFmfp4M6Gw04desMREREZEL2DQflv3RuJ8wAfr/Q6FYNXy7O4vTRWU0D/DiitbNzC5HRESkyhSMNRJ3JpQ34d98VE34RURERM615UP45AHADn3+ADfPUChWTUvKd6P8TXwELi76sxMREeehYKyR6Nu6GTHNfDhTXMZnW9WEX0RERASArR/D0gmAHXr9DgY+r1Csmk7ml/DtniwAhnWPMrkaERGR6lEw1kgYTfiNWWPvJ6eaXI2IiIhIPXEmA+w26DkOBr2gUOwSfLbtGKVWO52a+9M+vInZ5YiIiFSLgrFGZHjP8ib8qafYcVRN+EVERES48kH47WK45V/gokvjS7G0fBmlmu6LiIgz0t/+jUiwnyf91YRfREREpLI2NyoUu0RHjuez4chJXCxGfzERERFnoyuARubO8uWUSzcdpaBETfhFRERE5NIt3WT0ru3XJpgwfy+TqxEREak+BWONTKUm/FuOmV2OiIiIiDgpu93Okk1pgJZRioiI81Iw1si4uFi4w9GEX8spRUREROTSbEo9xeHjBXi7uzKgvF2HiIiIs1Ew1giNKG/Cvzn1FDuP5pldjoiIiIg4oYqm+wM6h+Hr6WZyNSIiIpdGwVgjFOznSf9Oxrd6H67TrDERERERqZ6SMhufbjH6iw3tEWVyNSIiIpdOwVgjNbp8OeWSjelqwi8iIiIi1fL93mxOFpQS7OdJv9hmZpcjIiJyyRSMNVJXxjajRVMfTheX8dlWNeEXERERkapbUr6McnB8BG6u+pVCREScl/4Wa6RcXCyOWWMfqAm/iIiIiFRRbmEpK3ZlAtqNUkREnJ+CsUZsRM8o3FwsbEo5xa5jasIvIiIiIr9u+fZjlJTZaBvqR+cIf7PLERERuSwKxhqxkCae9O8cBsCHmjUmIiIiIlWweKOxjHJoj0gsFovJ1YiIiFweBWONXMVyysWb0ikssZpcjYiIiIjUZ+mnCkk6dAKAwfFaRikiIs5PwVgj1y822GjCX1TGZ1uPml2OiIiIiNRjS8ub7l/RuimRgd4mVyMiInL5LikYe/XVV4mJicHLy4uEhASSk5MvOnbOnDlcffXVBAUFERQURGJi4nnj7XY7U6dOpXnz5nh7e5OYmMi+ffsupTSpJhcXC3f0iQbUhF9ERERELs5utzt2o1TTfRERaSiqHYwtWLCASZMm8cQTT7Bx40bi4uIYMGAAWVlZFxy/atUqRo8ezbfffsuaNWuIjo6mf//+pKenO8Y8//zz/Pvf/2b27NkkJSXh6+vLgAEDKCoquvR3JlVW0YR/Y8opdmeoCb+IiEhjVp0vQEtLS5k+fTqxsbF4eXkRFxfH8uXLK42ZNm0aFoul0q1Dhw61/TakFuw4msf+rDN4urkwsGtzs8sRERGpEdUOxmbOnMm9997L+PHj6dSpE7Nnz8bHx4e5c+decPz8+fO5//77iY+Pp0OHDrz55pvYbDZWrlwJGN88zZo1iylTpjB48GC6devGu+++y9GjR1m6dOllvTmpmtAmXtzUqaIJf6rJ1YiIiIhZqvsF6JQpU3jjjTd4+eWX2blzJ/fddx9Dhw5l06ZNlcZ17tyZY8eOOW6rV6+ui7cjNayi6X5ipzD8vdxNrkZERKRmVCsYKykpYcOGDSQmJp49gYsLiYmJrFmzpkrnKCgooLS0lKZNmwJw6NAhMjIyKp0zICCAhISEKp9TLp+jCf/GNDXhFxERaaSq+wXovHnz+Pvf/86gQYNo3bo1EyZMYNCgQbz44ouVxrm5uREeHu64BQcH18XbkRpUZrWxbIvRj3aomu6LiEgDUq1gLCcnB6vVSlhYWKXjYWFhZGRkVOkcf/vb34iIiHAEYRXPq845i4uLycvLq3STy3NVm2CigrzJKyrj823HzC5HRERE6tilfAFaXFyMl5dXpWPe3t7nzQjbt28fERERtG7dmrvuuouUFPU1dTar9+eQc6aYIB93rm0fYnY5IiIiNaZOd6V89tln+fDDD1myZMl5F1HVMWPGDAICAhy36OjoGqyycXJxsThmjakJv4iISONzKV+ADhgwgJkzZ7Jv3z5sNhsrVqxg8eLFHDt29ku2hIQE3n77bZYvX87rr7/OoUOHuPrqqzl9+vRFa9GXoPVPRdP92+IicHfVxvYiItJwVOtvteDgYFxdXcnMzKx0PDMzk/Dw8F987gsvvMCzzz7L//73P7p16+Y4XvG86pxz8uTJ5ObmOm6pqeqLVRNG9jKa8G84cpI9GRe/WBUREREBeOmll2jbti0dOnTAw8ODiRMnMn78eFxczl5iDhw4kJEjR9KtWzcGDBjAF198walTp/joo48uel59CVq/nCku46sdRjiq3ShFRKShqVYw5uHhQc+ePR2N8wFHI/2+ffte9HnPP/88Tz31FMuXL6dXr16VftaqVSvCw8MrnTMvL4+kpKSLntPT0xN/f/9KN7l8oU28SOxofEusWWMiIiKNy6V8ARoSEsLSpUvJz8/nyJEj7N69Gz8/P1q3bn3R1wkMDKRdu3bs37//omP0JWj98tX2DIpKbbQK9iU+OtDsckRERGpUtedBT5o0iTlz5vDOO++wa9cuJkyYQH5+PuPHjwdgzJgxTJ482TH+ueee4/HHH2fu3LnExMSQkZFBRkYGZ86cAcBisfDnP/+Zf/zjHyxbtoxt27YxZswYIiIiGDJkSM28S6my0Qlnm/AXlaoJv4iISGNxqV+AAnh5eREZGUlZWRmLFi1i8ODBFx175swZDhw4QPPmzS86Rl+C1i9LNxvLKIfER2KxWEyuRkREpGa5VfcJo0aNIjs7m6lTp5KRkUF8fDzLly939KNISUmpNH3+9ddfp6SkhBEjRlQ6zxNPPMG0adMAeOSRR8jPz+cPf/gDp06d4qqrrmL58uWX1YdMLs3VbYKJDPQm/VQhX2w7xrAeUWaXJCIiInVk0qRJjB07ll69etGnTx9mzZp13hegkZGRzJgxA4CkpCTS09OJj48nPT2dadOmYbPZeOSRRxznfPjhh7ntttto2bIlR48e5YknnsDV1ZXRo0eb8h6lejLzivhxfw6gZZQiItIwVTsYA5g4cSITJ0684M9WrVpV6fHhw4d/9XwWi4Xp06czffr0SylHapDRhD+aF/63l/eTUhSMiYiINCLV/QK0qKiIKVOmcPDgQfz8/Bg0aBDz5s0jMDDQMSYtLY3Ro0dz/PhxQkJCuOqqq1i7di0hIdrZ0Bl8sjkdmx16tgyiRTMfs8sRERGpcRa73W43u4jLlZeXR0BAALm5uZpqXwMy84q48tlvsNrs/O8v19AurInZJYmIiNQKXUM4B31O5hn40g/sOpbHP4Z04bdXtDS7HBERkSqr6vWD9lqW84T5e5HYMRRQE34RERGRxmp3Rh67juXh7mrh1m4X7wknIiLizBSMyQWN7lPRhD9dTfhFREREGqElm4ym+9e3DyXQx8PkakRERGqHgjG5oKvbhhAZ6E1uYSlfbj9mdjkiIiIiUoesNjufbDoKwLAearovIiINl4IxuSBXFwt39I4G4IOkVJOrEREREZG6tPbgcTLyivD3cuP6DqFmlyMiIlJrFIzJRY3sFY2ri4XkwyfYl3na7HJEREREpI5ULKO8pVsEnm6uJlcjIiJSexSMyUWFB3hxQ4eKJvyaNSYiIiLSGBSWWPlym9FKQ8soRUSkoVMwJr/ozgSjCf+ijWlqwi8iIiLSCKzYlUl+iZWoIG96tggyuxwREZFapWBMftE15zThX749w+xyRERERKSWLdmYBsDQ7pG4uFhMrkZERKR2KRiTX+TqYmFUeRP+95NTTK5GRERERGpT9ulivt+XA8CQ7lpGKSIiDZ+CMflVt/eKxsUCyYdOsD/rjNnliIiIiEgt+WzrUaw2O3FRAcSG+JldjoiISK1TMCa/ymjCHwbAB5o1JiIiItJgVexGOVSzxUREpJFQMCZVcmeCsZxSTfhFREREGqb9WWfYmpaLq4uFW+MizC5HRESkTigYkyq5tl0oEQFenCoo5asdasIvIiIi0tAsLZ8tdm27EIL9PE2uRkREpG4oGJMqMZrwtwDg/SQtpxQRERFpSGw2O0s3G8GYmu6LiEhjomBMquz23lG4WCDp0AkOZKsJv4iIiEhDsf7ISdJOFuLn6Ub/TmFmlyMiIlJnFIxJlTUP8OaGDqEAfKBZYyIiIiINxpJNaQAM7BKOl7urydWIiIjUHQVjUi2j+xjLKdWEX0RERKRhKCq18tnWY4B2oxQRkcZHwZhUy7XtQmge4MVJNeEXERERaRC+3Z3F6aIymgd4cUXrZmaXIyIiUqcUjEm1uLm6MKp3NAAfJGs5pYiIiIizW1K+G+Vv4iNwcbGYXI2IiEjdUjAm1XZ7r2hcLLD24AkOqgm/iIiIiNM6mV/Ct3uyABjWPcrkakREROqegjGptohAb65vbzTh/3BdqsnViIiIiMil+mzbMUqtdjo196d9eBOzyxEREalzCsbkklQ04V+4IY3iMjXhFxEREXFGS8uXUarpvoiINFYKxuSSXNc+hHB/L07kl/DVjkyzyxERERGRajpyPJ8NR07iYjH6i4mIiDRGCsbkkri5unB7RRP+JDXhFxEREXE2SzcdBaBfm2DC/L1MrkZERMQcCsbkko3qbTThX3PwuJrwi4iIiDgRu93Okk1pgJZRiohI46ZgTC5ZZKA315U34V+gJvwiIiIiTmNT6ikOHy/A292VAZ3DzS5HRETENArG5LJUNOH/WE34RURERJxGRdP9AZ3D8PV0M7kaERER8ygYk8tyffsQwvw9OZFfwv/UhF9ERESk3isps/HpFqO/2NAeUSZXIyIiYi4FY3JZ3FxdGNWrvAl/sprwi4iIiNR33+/N5mRBKcF+nvSLbWZ2OSIiIqZSMCaX7fbe0Vgs8NOB4xzKyTe7HBERERH5BUvKl1EOjo/AzVW/DoiISOOmvwnlskUF+XBduxAAPlynWWMiIiIi9VVuYSkrdhntL7QbpYiIiIIxqSEVTfgXrk+jpMxmcjUiIiIiciHLtx+jpMxG21A/Okf4m12OiIiI6RSMSY24oUMooU08OZ5fwoqdasIvIiLirF599VViYmLw8vIiISGB5OTki44tLS1l+vTpxMbG4uXlRVxcHMuXL7/o+GeffRaLxcKf//znWqhcqmLxRmMZ5dAekVgsFpOrERERMZ+CMakRbq4ujOptNOF/P/mIydWIiIjIpViwYAGTJk3iiSeeYOPGjcTFxTFgwACysrIuOH7KlCm88cYbvPzyy+zcuZP77ruPoUOHsmnTpvPGrlu3jjfeeINu3brV9tuQi0g/VUjSoRMADI7XMkoRERFQMCY16PZeRhP+H/cf57Ca8IuIiDidmTNncu+99zJ+/Hg6derE7Nmz8fHxYe7cuRccP2/ePP7+978zaNAgWrduzYQJExg0aBAvvvhipXFnzpzhrrvuYs6cOQQFBdXFW5ELWFredP+K1k2JDPQ2uRoREZH6QcGY1Jjopj5c07aiCX+qydWIiIhIdZSUlLBhwwYSExMdx1xcXEhMTGTNmjUXfE5xcTFeXl6Vjnl7e7N69epKxx544AFuueWWSueWumW32x27UarpvoiIyFkKxqRG3ZlQ3oR/Q6qa8IuIiDiRnJwcrFYrYWFhlY6HhYWRkZFxwecMGDCAmTNnsm/fPmw2GytWrGDx4sUcO3bMMebDDz9k48aNzJgxo8q1FBcXk5eXV+kml2fH0Tz2Z53B082FgV2bm12OiIhIvaFgTGpURRP+nDMlfL1LTfhFREQaspdeeom2bdvSoUMHPDw8mDhxIuPHj8fFxbjETE1N5U9/+hPz588/b2bZL5kxYwYBAQGOW3R0dG29hUajoul+Yqcw/L3cTa5GRESk/lAwJjXK3dWF23uVN+FPSjG5GhEREamq4OBgXF1dycys/MVWZmYm4eHhF3xOSEgIS5cuJT8/nyNHjrB79278/Pxo3bo1ABs2bCArK4sePXrg5uaGm5sb3333Hf/+979xc3PDarVe8LyTJ08mNzfXcUtNVYuGy1FmtbFsy1EAhqrpvoiISCUKxqTGjeptNOFfvT+HI8fVhF9ERMQZeHh40LNnT1auXOk4ZrPZWLlyJX379v3F53p5eREZGUlZWRmLFi1i8ODBANx4441s27aNzZs3O269evXirrvuYvPmzbi6ul7wfJ6envj7+1e6yaVbvT+HnDPFBPm4c237ELPLERERqVfczC5AGp7opj5c3TaE7/dm8+G6VP52cwezSxIREZEqmDRpEmPHjqVXr1706dOHWbNmkZ+fz/jx4wEYM2YMkZGRjn5hSUlJpKenEx8fT3p6OtOmTcNms/HII48A0KRJE7p06VLpNXx9fWnWrNl5x6X2VDTdvy0uAndXfS8uIiJyLgVjUivu7NOC7/dm8/H6VP6S2A4PN12EiYiI1HejRo0iOzubqVOnkpGRQXx8PMuXL3c05E9JSXH0DwMoKipiypQpHDx4ED8/PwYNGsS8efMIDAw06R3Iz50pLuOrHcbmCdqNUkRE5HwWu91uN7uIy5WXl0dAQAC5ubmaal9PlFptXPnsN2SfLub1u3po9yMREamXdA3hHPQ5XbpFG9J46OMttAr25ZuHrsVisZhdkoiISJ2o6vWDpvFIrTCa8EcB8H6ymvCLiIiImGHpZmMZ5ZD4SIViIiIiF6BgTGrNHb1bAPDDvhxSjheYXI2IiIhI45KZV8SP+3MALaMUERG5GAVjUmuMJvzBAHy4TrPGREREROrSJ5vTsdmhZ8sgWjTzMbscERGReknBmNSqO/sYs8Y+Wp9GqdVmcjUiIiIijceSTUcBzRYTERH5JQrGpFYldgoj2M+TnDPFrNyVaXY5IiIiIo3C7ow8dh3Lw93Vwq3dtAmSiIjIxSgYk1pVuQl/qsnViIiIiDQOSzYZTfevbx9KoI+HydWIiIjUXwrGpNadbcKfTeoJNeEXERERqU1Wm51PypdRDuuhZZQiIiK/RMGY1LoWzYwm/Ha7mvCLiIiI1La1B4+TkVeEv5cb13cINbscERGRek3BmNSJ0WrCLyIiIlInKpZR3tItAk83V5OrERERqd8UjEmdSOwYRrCfB9mni1m5K8vsckREREQapMISK19uOwZoGaWIiEhVKBiTOuHh5sLIXtEAfJCs5ZQiIiIitWHFrkzyS6xEBXnTs0WQ2eWIiIjUewrGpM7c0dsIxr5XE34RERGRWrFkYxoAQ7tH4uJiMbkaERGR+u+SgrFXX32VmJgYvLy8SEhIIDk5+aJjd+zYwfDhw4mJicFisTBr1qzzxkybNg2LxVLp1qFDh0spTeqxls18uaqN0YT/o/WpZpcjIiIi0qBkny7m+305AAzprmWUIiIiVVHtYGzBggVMmjSJJ554go0bNxIXF8eAAQPIyrpw36iCggJat27Ns88+S3h4+EXP27lzZ44dO+a4rV69urqliROoaMK/YF2qmvCLiIiI1KDPth7FarMTFxVAbIif2eWIiIg4hWoHYzNnzuTee+9l/PjxdOrUidmzZ+Pj48PcuXMvOL53797885//5I477sDT0/Oi53VzcyM8PNxxCw4Orm5p4gRu6hRGM18Psk4X881uNeEXERERqSkVu1EO1WwxERGRKqtWMFZSUsKGDRtITEw8ewIXFxITE1mzZs1lFbJv3z4iIiJo3bo1d911FykpatDeEHm4uTCiVxSgJvwiIiIiNWV/1hm2puXi6mLh1rgIs8sRERFxGtUKxnJycrBarYSFhVU6HhYWRkZGxiUXkZCQwNtvv83y5ct5/fXXOXToEFdffTWnT5++4Pji4mLy8vIq3cR5jO5tLKf8bm82aSfVhF9ERETkci0tny12bbsQgv0uvkpDREREKqsXu1IOHDiQkSNH0q1bNwYMGMAXX3zBqVOn+Oijjy44fsaMGQQEBDhu0dHRdVyxXI6YYF/6tWlmNOFfpyb8IiIiIpfDZrM7llGq6b6IiEj1VCsYCw4OxtXVlczMzErHMzMzf7GxfnUFBgbSrl079u/ff8GfT548mdzcXMctNVXhirNxNOFfn0qZmvCLiIiIXLL1R06SfqoQP083+ncK+/UniIiIiEO1gjEPDw969uzJypUrHcdsNhsrV66kb9++NVbUmTNnOHDgAM2bN7/gzz09PfH39690E+fSv1M4zXw9yMxTE34RERGRy7FkUxoAA7uE4+XuanI1IiIizqXaSyknTZrEnDlzeOedd9i1axcTJkwgPz+f8ePHAzBmzBgmT57sGF9SUsLmzZvZvHkzJSUlpKens3nz5kqzwR5++GG+++47Dh8+zE8//cTQoUNxdXVl9OjRNfAWpT7ycHNhRE814RcRERG5HEWlVj7begzQbpQiIiKXwq26Txg1ahTZ2dlMnTqVjIwM4uPjWb58uaMhf0pKCi4uZ/O2o0eP0r17d8fjF154gRdeeIFrr72WVatWAZCWlsbo0aM5fvw4ISEhXHXVVaxdu5aQkJDLfHtSn93RpwVvfH+QVXuzST9VSGSgt9kliYiIiDiVb3dncbqojOYBXlzRupnZ5YiIiDidagdjABMnTmTixIkX/FlF2FUhJiYGu93+i+f78MMPL6UMcXKtgn25MrYZPx04zoJ1qUy6qZ3ZJYmIiIg4lcXlTfd/Ex+Bi4vF5GpEREScT73YlVIar4om/B+tUxN+ERERkeo4mV/Cqj1Gr9Zh3aNMrkZERMQ5KRgTU/XvHEZTXw8y8or4dk+22eWIiIiIOI3Pth2j1GqnY3N/2oc3MbscERERp6RgTEzl6eaqJvwiIiIil2Bp+TLKYWq6LyIicskUjFXF6Uz4aAzk55hdSYN0R+9oAFbtySL9VKHJ1YiIiIjUf0eO57PhyElcLEZ/MREREbk0CsaqYskfYOcn8NYgOJ1hdjUNTusQP65o3RSb3eg1JiIiIiK/bOmmowD0axNMmL+XydWIiIg4LwVjVTHoRfCPhJw98NZAOKXwpqbdmdASgI/Wqwm/iIiIyC+x2+0s2ZQGwFAtoxQREbksCsaqIrgNjP8CAlvAiYPGzLETB82uqkEZ0DmMIB93juUW8d1eNeEXERERuZhNqac4fLwAb3dXBnQON7scERERp6ZgrKqCYmD8cmjWBnJTjHAse6/ZVTUY5zbhfz9JTfhFRERELqai6f6AzmH4erqZXI2IiIhzUzBWHQGRMO4LCOkIp4/B24Mgc4fZVTUYd/RpAcC3e7I4qib8IiIiIucpKbPx6Rajv9jQHlEmVyMiIuL8FIxVV5MwGPc5hHeD/Gx4+xY4usnsqhqE2BA/ElqVN+Ffrz5uIiIiZnj11VeJiYnBy8uLhIQEkpOTLzq2tLSU6dOnExsbi5eXF3FxcSxfvrzSmNdff51u3brh7++Pv78/ffv25csvv6ztt9Fgfb83m5MFpQT7edIvtpnZ5YiIiDg9BWOXwrcZjF0Gkb2g8CS88xtISTK7qgbhzgRj1tiCdalYbXaTqxEREWlcFixYwKRJk3jiiSfYuHEjcXFxDBgwgKysrAuOnzJlCm+88QYvv/wyO3fu5L777mPo0KFs2nT2S8OoqCieffZZNmzYwPr167nhhhsYPHgwO3Zo1v2lWFK+jHJwfARurrqUFxERuVwWu93u9OlDXl4eAQEB5Obm4u/vX3cvXHwa3h8FR34Ed1+4cwG0urruXr8BKiq10nfGSk4WlDJ3XC9u6BBmdkkiItKAmXYNUU8lJCTQu3dvXnnlFQBsNhvR0dE8+OCDPProo+eNj4iI4LHHHuOBBx5wHBs+fDje3t689957F32dpk2b8s9//pPf/e53VapLn5Mht7CU3k9/TUmZjc8evIoukQFmlyQiIlJvVfX6QV8zXQ7PJnDXQmh9PZTmw/wRsP9rs6tyal7urgzvUdGEX8spRURE6kpJSQkbNmwgMTHRcczFxYXExETWrFlzwecUFxfj5eVV6Zi3tzerV6++4Hir1cqHH35Ifn4+ffv2vWgtxcXF5OXlVboJLN9+jJIyG21D/egc0XgDQhERkZqkYOxyefjA6A+h3c1QVgQfjIbdX5hdlVOraML/ze5MjuWqCb+IiEhdyMnJwWq1EhZWebZ2WFgYGRkZF3zOgAEDmDlzJvv27cNms7FixQoWL17MsWPHKo3btm0bfn5+eHp6ct9997FkyRI6dep00VpmzJhBQECA4xYdHX35b7ABWLzRWEY5tEckFovF5GpEREQaBgVjNcHdC26fBx1/A9YS+Ohu2L7Y7KqcVptQP/pUNOFfl2Z2OSIiInIRL730Em3btqVDhw54eHgwceJExo8fj4tL5UvM9u3bs3nzZpKSkpgwYQJjx45l586dFz3v5MmTyc3NddxSUzWLPP1UIUmHTgAwOD7S5GpEREQaDgVjNcXNA0a8BV1vB1sZLPodbP7A7Kqc1p19Kprwp6gJv4iISB0IDg7G1dWVzMzMSsczMzMJDw+/4HNCQkJYunQp+fn5HDlyhN27d+Pn50fr1q0rjfPw8KBNmzb07NmTGTNmEBcXx0svvXTRWjw9PR27WFbcGrul5U33r2jdlMhAb5OrERERaTgUjNUkVzcYOht6jAG7DZbeB+vfMrsqp3Rzl3ACfdw5mlvE93uzzS5HRESkwfPw8KBnz56sXLnSccxms7Fy5cpf7AcG4OXlRWRkJGVlZSxatIjBgwf/4nibzUZxcXGN1N0Y2O12x26UQ7trtpiIiEhNUjBW01xc4daXoM8fjMef/RnWvm5qSc6oUhP+5BSTqxEREWkcJk2axJw5c3jnnXfYtWsXEyZMID8/n/HjxwMwZswYJk+e7BiflJTE4sWLOXjwID/88AM333wzNpuNRx55xDFm8uTJfP/99xw+fJht27YxefJkVq1axV133VXn789Z7Tiax/6sM3i6uTCwa3OzyxEREWlQ3MwuoEFycYGBz4ObF/z0b1j+KJQWwtWTzK7MqYzuE81/Vx/im91ZZOQWER7g9etPEhERkUs2atQosrOzmTp1KhkZGcTHx7N8+XJHQ/6UlJRK/cOKioqYMmUKBw8exM/Pj0GDBjFv3jwCAwMdY7KyshgzZgzHjh0jICCAbt268dVXX3HTTTfV9dtzWhVN9xM7heHv5W5yNSIiIg2LxW63O30Dp7y8PAICAsjNza1fPSjsdlj1LHz3rPH4mkfg+r+DdhGqsttnryH58Akm3dSOP97Y1uxyRESkgam31xBSSWP+nMqsNq6Y8Q05Z4p5c0wvEjuF/fqTREREpMrXD1pKWZssFrh+MiROMx5//zyseNwIzKRKRicY27MvWJeqJvwiIiLS6Kzen0POmWKCfNy5tn2I2eWIiIg0OArG6sJVf4GbnzPu//QyfPFXsNnMrclJDOzSnABvd9JPFfLFtmNmlyMiIiJSpyqa7t8WF4G7qy7dRUREapr+dq0rV9wHt84CLLBuDnz6R7BZza6q3ju3Cf+DH2zi9++sY1tarslViYiIiNS+M8VlfLUjA9BulCIiIrVFwVhd6jUehs4GiwtsmgdL/g+sZWZXVe9N6t+Ood0jcbHA17uyuO2V1fzu7XVsTTtldmkiIiIitear7RkUldpoFexLfHSg2eWIiIg0SArG6lrcHTBiLri4wbaPYeE4KCsxu6p6zc/TjX+NiufrSdcyrDwgW7k7i9+88qMCMhEREWmwlm42llEOiY/Eos2bREREaoWCMTN0Hgqj3gNXD9j1KSz4LZQWmV1Vvdc6xI+ZFwnI7nl7HVtST5ldooiIiEiNyMwr4sf9OYCWUYqIiNQmBWNmaT8QRn8Ibt6w7yv4YBSU5JtdlVOoFJD1MAKyb3ZnMfhVBWQiIiLSMHyyOR2bHXq2DKJFMx+zyxEREWmwFIyZqc2N8NuF4O4LB1fBeyOgKM/sqpxG6xA/Zt4ez8qHrjsvIBv/VjKbFZCJiIiIk1qy6Sig2WIiIiK1TcGY2WKugjFLwTMAUn6CeUOg8KTZVTmVVsG+joBseI8oXCzw7Z5shiggExERESe0OyOPXcfycHe1cGu35maXIyIi0qApGKsPovvA2GXgHQTpG+Cd2yA/x+yqnE6rYF9evD3OEZC5ulgcAdm4t5LZlKLAUUREROq/JZuMpvvXtw8l0MfD5GpEREQaNgVj9UVEPIz7HHxDIGMbvH0LnM4wuyqn5AjIJl3LiJ5GQLZqTzZDX/tJAZmIiIjUa1abnU/Kl1EO66FllCIiIrVNwVh9EtYZxn8JTSIgeze8NQhy08yuymnFBPvywsgLB2Rj5yazUQGZiIiI1DNrDx4nI68Ify83ru8QanY5IiIiDZ6CsfomuC2M/wICW8CJA/DWQDh52OyqnFpFQPbNQ9cysjwg+25vNsMUkImIiEg9U7GM8pZuEXi6uZpcjYiISMOnYKw+atrKmDnWtDWcSoG5AyFnv9lVOb2WzXz550UCsjFzk9lwRAGZiIiImKewxMqX244BWkYpIiJSVxSM1VcBUUY4FtIBTh81Zo5l7jS7qgbh3IDs9l5GQPb93myGv66ATERERMyzYlcm+SVWooK86dkiyOxyREREGgUFY/VZk3CjIX9YV8jPMhryH91sdlUNRstmvjw/Io5vH7ruvIDs7v8mKSATERGROrVko9Fbdmj3SFxcLCZXIyIi0jgoGKvvfINh3KcQ2RMKT8A7v4HUdWZX1aC0aObjCMhG9YrG1cXCD/tyzgnITphdooiIiDRw2aeL+X5fDgBDumsZpYiISF1RMOYMvIPg7qXQoi8U58K8IXD4R7OranBaNPPhuRHdHAGZmyMgW6OATERERGrVZ1uPYrXZiYsKIDbEz+xyREREGg0FY87Cyx9+uwhaXQslZ+C94XDgG7OrapAcAdnD13FHbwVkIiIiUvsqdqPUbDEREZG6pWDMmXj4wp0LoG1/KCuE90fBni/NrqrBim7qw7PDLx6QrT+sgExEREQu3/6sM2xNy8XVxcJtcRFmlyMiItKoKBhzNu7eMGo+dLwNrCWw4LewY4nZVTVo5wZko/ucDchGzF7Db99UQCYiIiKXZ2n5bLFr24UQ7OdpcjUiIiKNi4IxZ+TmASPehq4jwVYGC++BLQvMrqrBi27qw4xhlQOy1fvPBmTrFJCJiIhINdlsdi2jFBERMZGCMWfl6gZD34DuvwW7DZb8H2x4x+yqGoXKAVkLR0A2cvYa7npzrQIyERERqbL1R06SfqoQP083buoYZnY5IiIijY6CMWfm4gq3vQy9fw/Y4dM/QtIbZlfVaBgBWddKAdmP+487ArLkQwrIRERE5Jct2ZQGwM1dwvH2cDW5GhERkcZHwZizc3GBQS9A34nG4y8fgdWzTC2psakIyFb99TruTGiBu6sRkN3+xhrunKOATERERC6sqNTKZ1uPATBMyyhFRERMoWCsIbBYoP8/4JpHjMdfPwGrngW73dy6GpmoIB+eGWrMIKsIyH46cDYgSzp43OwSRUREpB75dncWp4vKaB7gxRWtm5ldjoiISKOkYKyhsFjghsfgxqnG41Uz4OtpCsdMUBGQrfrr9dx1TkA26j9rGf0fBWQiIiJiWFzedP838RG4uFhMrkZERKRxUjDW0Fz9EAyYYdz/cRZ8+Tew2UwtqbGKDPTm6Z8FZGsOng3I1iogExERabRO5pewak8WAMO6R5lcjYiISOOlYKwh6ns/3Pov437yG/DZn8BmNbemRuzcgOy3V5wNyO74z1ru+M8aBWQiIiKN0GfbjlFqtdOxuT/tw5uYXY6IiEijpWCsoep1Dwx5HSwusPFdWDoBrGVmV9WoRQZ6848hlQOytQdPOAKyNQcUkImIiDQWS8uXUarpvoiIiLkUjDVk8XfC8DfB4gpbF8Cie6CsxOyqGr2KgOy7v17P3Ve0xMPVhbUHTzB6zlpGvaGATEREpKE7cjyfDUdO4mIx+ouJiIiIeRSMNXRdhsOoeeDqATs/gY/GQGmR2VUJEBHozVNDurDqr9c5ArKkQwrIREREGrol5bPF+rUJJszfy+RqREREGjcFY41Bh1vgjg/AzQv2fgkf3AElBWZXJeV+KSC7/Y01/HQgB7t2FxURkTry6quvEhMTg5eXFwkJCSQnJ190bGlpKdOnTyc2NhYvLy/i4uJYvnx5pTEzZsygd+/eNGnShNDQUIYMGcKePXtq+23UW3a73bGMcqiWUYqIiJhOwVhj0TYR7voY3H3h4LcwfwQUnza7KjlHRUD23SPXMaavEZAlHzrBnXOSGPWftQrIRESk1i1YsIBJkybxxBNPsHHjRuLi4hgwYABZWVkXHD9lyhTeeOMNXn75ZXbu3Ml9993H0KFD2bRpk2PMd999xwMPPMDatWtZsWIFpaWl9O/fn/z8/Lp6W/XKptRTHD5egLe7KwM6h5tdjoiISKNnsTeA37Tz8vIICAggNzcXf39/s8up31KSykOxPIjqDXctBO9As6uSCziWW8jrqw7wYXIqJVYbAH1imvLnxLb0jW2GxWIxuUIREeena4jKEhIS6N27N6+88goANpuN6OhoHnzwQR599NHzxkdERPDYY4/xwAMPOI4NHz4cb29v3nvvvQu+RnZ2NqGhoXz33Xdcc801VaqrIX1OUz/ZzrtrjjAkPoJZd3Q3uxwREZEGq6rXD5ox1ti0SIAxn4BXIKStg3dug3z1sqqPmgd4M32wMYNsbMUMssMnuPPNJEa9sZaf9msGmYiI1JySkhI2bNhAYmKi45iLiwuJiYmsWbPmgs8pLi7Gy6tyjyxvb29Wr1590dfJzc0FoGnTphcdU1xcTF5eXqVbQ1BSZuPTLUcBGNojyuRqREREBC4xGKtO74kdO3YwfPhwYmJisFgszJr1/+3dd3hUZfr/8fek9wJpBJJA6AgkEIrICihoAEUBXZHVBaKyqwv+lm/WhrIKNnRVFhQUFxewLIoFsIMYBBTpEIr0GgiEJEAS0svM748TJoQgECA5KZ/XdZ3LzJkzZ+45MyYP99zP/Uy96nPKVWrcGUZ9Cx4BkLIV3r8dzpwwOyr5HY183Zn0Owmye95dzSolyERE5BpIT0+npKSE4ODgcvuDg4NJSUm54GNiY2OZMmUKe/fuxWq1snTpUhYsWMDx48cveLzVamXcuHH07NmT9u3b/24skydPxtfX176FhYVd+QurQVbsSeN0bhEBXq70bN7Q7HBERESEK0iMVbb3RG5uLpGRkbzyyiuEhFy4j0JlzynXQEh7iPsevBtB6g6YOxAyk82OSi7ibIJs5RM3MeqGprg4ObD+0GnuU4JMRERMMm3aNFq2bEmbNm1wcXFh7NixxMXF4eBw4SHmmDFj2L59O5988slFzzt+/HgyMzPt25EjR6oi/Gp3tun+ndGhODlq4oaIiEhNUOm/yFOmTGH06NHExcXRrl07Zs6ciYeHB7Nnz77g8V27duW1117j3nvvxdXV9ZqcU66RwFYQ9x34hsHJfTBnAJw+bHZUcgkhvm5MvOM6Vj5eMUH2x5mr+WWvEmQiIlJ5AQEBODo6cuJE+SryEydO/O6Xm4GBgSxatIicnBwOHz7Mrl278PLyIjIyssKxY8eO5ZtvvuGnn36iSZOLTyN0dXXFx8en3FbbZeYVsXSncW21GqWIiEjNUanE2JX0nqiKc9bVvhOmaBBpJMf8m0HGYSM5dnK/2VHJZTibIPv5nAqyDYdPc/9/lSATEZHKc3FxISYmhoSEBPs+q9VKQkICPXr0uOhj3dzcaNy4McXFxXzxxRfceeed9vtsNhtjx45l4cKFLFu2jGbNmlXZa6jJFm8/TmGxlZZBXlwXWvsTfSIiInVFpRJjV9J7oirOWVf7TpjGL9yYVhnQCrKSjeRY6k6zo5LLFOzz+wmyu2eu5ue9aUqQiYjIZYmPj2fWrFm8//777Ny5k0ceeYScnBzi4uIAGDFiBOPHj7cfv3btWhYsWMCBAwf4+eef6d+/P1arlSeeeMJ+zJgxY/joo4+YN28e3t7epKSkkJKSQl5eXrW/PjMt2GRMoxzSubFWlhYREalBamVzg7rad8JUPo1g1HcQ3B6yT8Dc2+D4VrOjkko4N0EW19NIkG08fJo//3edEmQiInJZhg0bxuuvv86zzz5LdHQ0iYmJLF682P4FZlJSUrnG+vn5+UyYMIF27doxZMgQGjduzC+//IKfn5/9mHfeeYfMzEz69OlDo0aN7Nv8+fOr++WZJjkjj7UHTwFwZ7SmUYqIiNQkTpU5+Ep6T1TFOV1dXX+3X5lcBa9AGPk1fDQUjm02Vqu8fyE0iTE7MqmEYB83nht0HQ/3bs7MFfuZtzbJniDrHO7HuH6tuLFlgL6tFhGRCxo7dixjx4694H3Lly8vd7t3797s2LHjoufTlzJlTfevj2xAYz93k6MRERGRc1WqYuxqek9U5znlKng0gBFfQlh3yM+ED+6Ew7+aHZVcgbMJsp+fuIkHejbD1cmBTUkZjJi9jrve+ZWVe1RBJiIiUtVsNhsLSxNjarovIiJS81R6KmVle08UFhaSmJhIYmIihYWFJCcnk5iYyL59+y77nFLN3Hzh/gXQ9EYoPAMf3QX7fzI7KrlCQT5uPDuo3QUTZEPf+ZUVSpCJiIhUmd+OZbEvNRtXJwcGdGhkdjgiIiJynkpNpQSj90RaWhrPPvssKSkpREdHV+g94eBQlm87duwYnTp1st9+/fXXef311+ndu7e9HP9S5xQTuHrBfZ/B/Pth348wbxgM+xBaxZodmVyhswmyh3tH8u7KA3y05jCbkzIYOXsdnUqnWPbSFEsREZFr6mzT/X7tgvFxczY5GhERETmfxVYHSkWysrLw9fUlMzMTHx8tf31NFRfAZ3Gw+1twcIa7Z0O7O8yOSq6B1DP5vLvCSJAVFFsBiA7zY1y/lvRuFagEmYjUCxpD1A619X0qLrFy/eRlpGcX8N6ILvRrpy99RUREqsvljh9q5aqUUo2cXOGe9+G6oWAtgs9GwdbPzI5KroEgbzf+eXs7fn7yJh76QzPcnB1IPJLBqDnrGfL2ryzfnaopliIiIlfhl33ppGcX4O/hTO/WgWaHIyIiIhegxJhcmqMz3PUeRN8HthJYMBo2fWh2VHKNBHm7MeH2dqx8omKCbPDbv/KTEmQiIiJX5GzT/UFRoTg7atgtIiJSE+kvtFweB0e4Yzp0eRCwwVdjYd0ss6OSa+hsguznJ25m9I1GgmzLkQziShNkc1cdZNvRTIpKrGaHKiIiUuNlFxSz5LcUQKtRioiI1GSVbr4v9ZiDA9z2Bji5wZoZ8N1jUJQHPf+f2ZHJNRTo7cozt7XjL72a85+V+/lwzWG2HMlgy5EMANydHYkK8yUmwp+YCH86h/vj5+FibtAiIiI1zJLtKeQXWWkW4El0mJ/Z4YiIiMjvUGJMKsdigdiXwNkNfn4Dlv4TivOh1+PGfVJnnJsg+3TDEdYfOsWmw6fJyi9mzYFTrDlwyn5s80BPukQ0MBJlEf5EBnji4KDPg4iI1F+LEo1plIOjG2tBGxERkRpMiTGpPIsF+j4Lzu6w7EX46SWjcqzvs0qO1UGB3q6MuakFAFarjf1p2Ww8fNrYkk5zIC2H/aXb/A1HAPDzcKZzeFlFWVSYLx4u+nUjIiL1w4msfFbtSwc0jVJERKSm079U5cr1ehyc3OGHZ+CXKUZyrP9kJcfqMAcHCy2DvWkZ7M293cIBOJVTyOak0/Zk2ZajGWTkFrFsVyrLdqUC4OhgoV0jH3tFWUyEP6G+bvoGXURE6qQvE5Ox2iAmwp/whh5mhyMiIiIXocSYXJ0bxhrTKr/9B6x9x5hWedsUox+Z1AsNPF3o2zaYvm2DASgqsbLjWJa9omzjodOkZOWzLTmTbcmZzP31EAAhPm7lEmXtGvng4qTPjYiI1H4LNx8DVC0mIiJSGygxJlev60NGQ/4vx8LGOUZy7I7p4KiPV33k7OhAVJgfUWF+PEAzAI5l5NkryjYlnea3Y1mkZOXz7bbjfLvtOACuTsbjYiL8iQk3EmYNPNXUX2qAjCPg7g+uXmZHIiK1wK6ULHYez8LZ0cLtHRuZHY6IiIhcgjIXcm10ut9Iji34C2z52EiODZ0Fjs5mRyY1QKifO6F+7gyKCgUgt7CYrUczjURZaWVZRm4R6w6eYt3Bsqb+kQGe9oqymAh/WgR6qam/VI+SItjxJax5B5I3GImxPk9Dlzj9XhORi1q42Wi6f1PrIK3aLCIiUgsoMSbXToe7wckVPouD3xZCcQH8ca6xT+QcHi5OXB/ZkOsjGwJgs9k4kJ5jT5RtOHyafanZHEjP4UB6Dp9vPAqAt5uTval/TIQ/UWF+eLnq15hcQ7mnjMrXde/BmWNl+/NOw/ePw/r3oP/L0KKfeTGKSI1VYrXxZek0yqGdNY1SRESkNrDYbDab2UFcraysLHx9fcnMzMTHx8fscGTvUph/v1E11rwvDPsIXNR4VionI7eQzUkZ9imYiUcyyCsqKXeMgwXahPjYE2UxEf408XdXU3+pvNSdRnXY1vnG7y4AzyBjqnjnEbDne2MV3tyTxn0tb4VbX4LAVubFLNeExhC1Q215n1btS+e+99bi4+bE+gn9cHVyNDskERGReutyxw9KjEnVOLACPr4XinKh6Y0w/BP155GrUlxiZVfKGXuibOPh0yRn5FU4Lsjb1Z4k6xzhz3WhPvqHiVyY1Qr7lhoJsQM/le0P6Qg9xsB1Q8pXvOZlwMrXYO27YC0CByfoOhr6PGlMtZRaSWOI2qG2vE+PfbaFzzceZXi3cCYP7WB2OCIiIvWaEmNivsOr4X9/hMIz0KQb3P85uPmaHZXUISmZ+WxKKkuU/XYsk6KS8r/SXJwc6NjY154o6xzuT6C3pvfWawXZkDgP1s6EU/uNfRYHaHM7XP8IhPeAi1UdntwPP0yA3d8Zt9394aZnICZOi47UQhpD1A614X3KKyyhy4tLySks4dO/9qBbswZmhyQiIlKvKTEmNUPyRvhwKORnQKNo+PNC8NBAUapGflEJ25KNpv4bDhkrYJ7KKaxwXERDD/vKlzER/rQK9sZRTf3rvtOHYd1/YNOHUJBp7HP1hc5/hm5/Af+Iyp1v/zJY/DSk7TRuB7aB2JehRd9rG7dUKY0haofa8D59teUY/+/jzTTxd2fl4zdpsRgRERGTKTEmNUfKNvjgTqM3T9B1MGIReAWZHZXUAzabjUMnc+0VZZsOn2ZP6hnO/63n5epEp3A/Oof706WpP9Fhfni7aeXBOsFmg8O/wtp3YNe3YLMa+xu2gO4PQ9Twq5vmXVIMm+bCspcgr3RF1Vb94dYXIaDlVYcvVU9jiNqhNrxPcXPW8dPuNB69uQX/uLW12eGIiIjUe0qMSc2SustIjmWnQEArGPEl+ISaHZXUQ5l5RSQeybAnyjYnnSansHxTf4sFWgd7l2vqH97AQ039a5PiAti+ANa8DSlby/Y3vxm6P2KsKungcO2eLy8DVvwL1r0L1mKj/1i3v0Lvx9V/rIbTGKJ2qOnvU9qZAq6fnECJ1UbCP3rTPFB9VUVERMymxJjUPCf3w/t3QNZR8G8KI78Gv3Czo5J6rsRqY3fKGTYmGYmyjYdPk3Qqt8JxAV4udA4vS5S1b+yLm7Oa+tc42amwYTas/y/kpBr7nNwhaphRIRbUtmqfP30f/PAM7Fls3HZvADc9rf5jV6m4xIqT4zVMZJ5DY4jaoaa/T3NWHWTS1zuIauLLl2P/YHY4IiIighJjUlOdPgwf3AGnD4FPExj5FTRsbnZUIuWknsln0+EMe2P/bUczKSyxljvG2dFC+8a+xJyTLAvycTMpYuH4FlgzE7Z/DiWlfeW8Q6HbaIgZVf29DfclwJJnyvqPBbUz+o81v6l646iFrFYb+9Ky2Zx0ms1JGWxOysDT1ZEFf+tZJc+nMUTtUNPfpzum/8LWo5k8N6gdcT2bmR2OiIiIoMSY1GRZx4zKsZN7wSvEmFYZ1MbsqER+V35RCb8dy7T3Ktt4OIP07IIKxzXxd6dLaZKsc4Q/rYO9q6zKRQBribEy5Jp34PCqsv1NuhqrS7a9AxxN7BVXUgwb58BPL5/Tf2wAxL6kLwTOcTK7gMQjRgJs85HTbD2SyZmC4nLHuDg6sG3Srbg6XfsqTY0haoea/D7tS82m35QVODpYWPt0XwK8tPKxiIhITaDEmNRs2anwwWBI/Q08AoyG/CEdzI5KzrJajZVEs1MhJ834OaSDMQVWsNlsHDmVx8akU/ZE2e6ULKzn/Tb1dHEkOtzPvgJmp3B/fN3V1P+q5WcaK0uuexcykox9Dk7QbrCREGvSxdTwKsg7DctfhfWzSvuPOUP3v0Kvx8Hdz+zoqlVhsZVdKVmllWCn2Xwkg8MnK05d9nBxpGMTXzqF+9MpzI/ocD+CvKumIlNjiNqhJr9Pry/ZzfSf9nFzmyBmj+pqdjgiIiJSSokxqflyT8GHg40pUG5+8OcF0DjG7KjqruICI8mVnQo56cbPOaU/n02A5aSX7bOVVDxHcAdoMxDa3AYhHY0u9QLAmfwithwprSpLOs3mw6crVL0AtAr2MirKSqdgNgvwVFP/y3VyP6ydCYnzoDDb2OfeALrEQdeHav6CHml74IcJsHeJcdujIdz0DHQeWSf7j9lsNo5n5rM5KYPEI8a0yG3JmRQUWysc2yLIi05hfnQKN1aFbRXsVW3VlhpD1A419X2yWm3c+K+fSM7I483hnbgjqob/HhIREalHlBiT2iEvA/73Rzi6Dly84b7PIKKH2VHVDjabUTmTk3ZOwuvc5FYaZKeV7SvIrPxzuPuDZyA4e0DKtvLJMt8wI0HW5jYIv6FO/sP+apRYbexLzWbj4dNsOHyKTYdPc+gClTENPF3oHO5H5wh/YsL9iQrzU1P/c9lscGC5kRDbswQo/ZMV2NaoDut4Dzi7mxlh5e37ERY/Dem7jdtB10H/lyGyj6lhXa28whK2JWeW9QY7cpoTWRWnHPu6O9Mp3I9OYf50CvcjKszP1EpKjSFqh5r6Pq07eIp73l2Nl6sT65/ph7uLfn+LiIjUFEqMSe1RkA0f3wuHfjYSMMM/gcjeZkdljpKiy6jmOifhZS2q3PkdnI1El1eg8V/PIPAMAK+g0tsBpftKfz63P1PuKdj7A+z6xmgsXnROksfdH1r1N5JkzW8GF89rcz3qmPTsAmPly9IVMLcczaTwvOoZJwcL153X1D/Etx429S/Kg63zjYb6ZxvYg/E56/6wkUSqzZV2JUWwYQ4sf9mYagnQeiDc+mKt6D9ms9k4dDK3XBJs5/EzlJw3n9jRwUKbEO9yibCaViWpMUTtUFPfp/ELtvLxuiPcHdOE1/8YZXY4IiIicg4lxqR2KcyF+ffB/mXg5AbDPoKWt5gd1dWz2YwpX5c1fTGt7B/IleHqW5rQukDCyzPwnKRXILj5XptkQlGeUcWz6xvY/T3kniy7z8kNIm8ykmStBxhxyAUVFlvtTf03JZ1mw6HTpJ6pWGHT2M+dto28aR7oRWSgJ80DvWge6IW/p4sJUVexrGOw/j0jaXS2Yb2zJ3S63+jLVQuSRpWSewpWvArrZhkVmQ7OcP3DRv8xN1+zo7PLzCtiyzkN8hOPZJCRWzExH+TtSudwIwHWKdyfDo19a3wFjcYQtUNNfJ/yi0ro+tKPnMkvZt5D3bmhhf7eiYiI1CRKjEntU1wAn40yVplzcIY/zoG2g8yOqiJriZEIKtevK/WcKY1pZT/npEFxfuXOb3E8p3Lr3OTWedVcXkHGwgXOJlcTWUvgyFrY9a2RKDt9qOw+iwOEXV825bKBlrC/GJvNRnJGnpEoK60s23GsYlP/s/w9nO1JsuZBnkQGeNE8yIswf/fatxrm0Q3G6pI7FhkN6gH8wqHbX42kWF1vUp+2G5Y8A/uWGrc9AuDm0v5jDtWbWCqx2thz4ky5Bvn7UrMrHOfi5ECHxr723mCdwv1o5OtWo6rBLofGELVDTXyfvt92nEf+t4lGvm6sevJmHBxq12dfRESkrlNiTGqnkiL44iHjH8cWRxj6H+hwd9U/b2FuWSVXuX5daRUTXrknsfc5ulzOnhep5jo34RVoTEt0qGVJjbNsNkjdWZYkO55Y/v6g68qSZI2iavdUuGqSU1DM1qOZ7Es9w/60HPanZXMgLYfkjLzffYyzo4WIhp40P6e6LDLQk8hAr5q1KmZJEez8ykiIHV1ftj/iD0bVVOuB1Z4UMt3epbDkaUjfY9wObg+xL1fp9PK0MwX2BNjmpNNsPZpJbmHFxTciGnqUS4K1CfHBxamW/q46h8YQtUNNfJ9Gf7CBpTtO8NfekYwf0NbscEREROQ8SoxJ7VVSDF+NhS0fAxa4c7pRMVIZVqsxLdE+dTHtvOmL5yW8inIqGaTFWFGu3PTFc7ZyPbsC62/PrYwjxlTLXd/AoV/KN+/3aVKWJIu4oXw/M7mk3MJiDqbnGMmy1Gx7wuxAejb5RRVX/Tsr0NuV5qVJMiNpZiTPGvu5V1+1Q+4p2DjXmD545pixz9EF2t9tJMQa1fM+PSVFsGE2/PQy5GcY+9rcDrc8f9VTSQuKS9hxLKt0SqSRCDt6umKS1cvViagwX3tfsOgwPxp6uV7Vc9dUGkPUDjXtfTqdU0i3l3+kqMTGknG9aB3ibXZIIiIich4lxqR2s1rh2/8z/vEMcNsbEH3/OQmtC628mFb+flvFioeLcnQt34+rXMLrvCb17g20CmNl5Z4yqmHszfvPSUa6+Z7TvL8vuHqZF2ctZ7XaOJaZx4HS6rL9adnsTzUSZhdaIfAsVycHmgV40jzIi+Zn/xvoRbMATzxdr9FnPXUXrH0HtsyH4tJkjGcgdH0Iujxg/P8lZXJPwfJXjJ5r9v5jj0Cvxy6r/5jNZuPo6Tx7AizxSAa/JWdRWFI+cWqxQKsgb6LD/Oy9wVoEeeFYT6aFaQxR0YwZM3jttddISUkhKiqKt956i27dul3w2KKiIiZPnsz7779PcnIyrVu35tVXX6V///72Y1auXMlrr73Gxo0bOX78OAsXLmTw4MGViqmmvU8frjnMPxdtp20jH77/+41mhyMiUitYrVYKCwvNDkPqEGdnZxwdf3+GiRJjUvvZbLD4KVg788rP4e7/O9VcARUTXi5emtpXXYry4eAKI0m26zvITS+7z9EVmpc27281wEhQyjVxJr/IXlW2P7UscXYoPbdCsuRcjXzd7NVlkef0NAvxuYx+UlYr7PsR1rwNB34q2x/SEa7/G7QfCk51sxLpmkndZUyv3J9g3PYMhJsnQKc/l5tqenba7eYjpStFJmWQnl0xGdrA06V0SqSRBOvYxBdvt/pbsakxRHnz589nxIgRzJw5k+7duzN16lQ+++wzdu/eTVBQxeT1k08+yUcffcSsWbNo06YNS5YsIT4+nl9//ZVOnToB8P3337Nq1SpiYmIYOnRonUiM3fXOr2w8fJpnBrZldK9Is8MREanxCgsLOXjwIFbr7485Ra6En58fISEhF/x3iRJjUjfYbLDsBfh5CmAzKibObT5fIel1ToWXR0NwqoOr9tU11hKjv9Sub2DnN3D64Dl3WiC8tHl/64F1b0XCGqLEauPo6dxy1WVnE2cnc37/Wz0PF8dyq2Se/blZgCdu1jxjOvTamXByn/EAi4PxXl7/NwjvoUR0ZdhsZf3HTu4FoKBhO1a3fIwf8lqxOSmD3SkVF2pwcrBwXahPaTWYMS0yvIFHrWuQX5U0hiive/fudO3alenTpwPGt/thYWE8+uijPPXUUxWODw0N5ZlnnmHMmDH2fXfddRfu7u589NFHFY63WCy1PjF2+GQOvV9bjoMFVo/vS7CPyYvgiIjUcDabjaSkJIqKiggNDcWhtvZTlhrFZrORm5tLamoqfn5+NGrUqMIxlzt+0FwwqdksFuj7LPQYa/zs5qd/TNc1Do5G8iv8erjlBUjbVVpJ9i0c2wxJq43thwkQ2LasL1loJ30WrhFHB6NZf0RDT25uU/6+jNxCe9P/s33M9qdlc/hkLrmFJWxPzmJ7cpb9+CaWNEY6/sC9Tj/hTS4AhU7epLcahssNf6Vh45ZKylyBjLwiNtui2dLqAxru/IA7Mz7A5+QO+px8gPySrvxc/CestmBCfd3sCbBO4X5cF+qLm3M9W8BArlhhYSEbN25k/Pjx9n0ODg7069eP1atXX/AxBQUFuLmVTwy5u7vzyy+/XFUsBQUFFBSUVTxmZWVd5OjqtXBzMgA9WwQoKSYichmKi4vJzc0lNDQUDw8Ps8OROsTd3R2A1NRUgoKCLjqt8mKUGJPawaOB2RFIdbBYIKitsfV6HDKPlm/en7bT2H5+HXwaG1VkbW6Dpn9Q8/4q4ufhQkyECzER/uX2FxZbSTqVy4G0bPanZmM9/Cudj39Mt4LVOJau2nrAGsKckv58kd+L3E1usGkv3m4Hy1WXnZ2iGdHQs06scHgtFJdY2ZVypqw3WFIGB9LPXSDkRqYQxWMuC7jX4Uf6O67nVudEcjv/Fa9+T4Kbqp7kyqSnp1NSUkJwcHC5/cHBwezateuCj4mNjWXKlCn06tWL5s2bk5CQwIIFCygpqWSfz/NMnjyZSZMmXdU5qoLNZmNRaWJsSKfGJkcjIlI7nP2b4OKi2Txy7Z1NthYVFSkxJiJ1kG8T6Dba2PIyzmne/yNkJcP6Wcbm5gstY40kWYu+4KrVwaqai5MDLRo40+LYT7DnHTi+xX5fYURvDjb/M5tcu+Kalkv3tGz2p+Vw9HQuZ/KLSTySQeKRjHLnc3SwEN7A45w+Zp6lCTQvGnjW7UHUiax8NieV9QXbmpxxwZVFIwM8iS7tC9YpzI/WIcNwPLkbFo/H4cBPeG2YDjs/hb7/hOj7yvUfE6kq06ZNY/To0bRp0waLxULz5s2Ji4tj9uzZV3Xe8ePHEx8fb7+dlZVFWFjY1YZ71TYfyeDQyVzcnR2JvS7E7HBERGoVzRqQqnAtPldKjIlI7eDuBx3/aGxF+XBwpZEk2/2dsRLptk+NzdEVInuX9SXTSofXXnYabJhtrJaYk2rsc3KDqHuh+8O4BLWlNdD6vIflF5Vw+GRu6ZTM7LIpmqnZ5BSWcDA9h4PpObAztdzj/D2cK/Qxax7kRZi/O06OtavKLL+ohN+OZdqTYJuTTnMsM7/Ccd5uTuX6gkU38cP/QgnCoLbw54Ww94fS/mP74KtHYd1/oP8rRjWlyGUKCAjA0dGREydOlNt/4sQJQkIunAQKDAxk0aJF5Ofnc/LkSUJDQ3nqqaeIjLy6hvSurq64uta8hTnOVovFXhd87VbsFREREVPpL7qI1D7ObtDqVmOz/huObijrS3Zqv5Ek2PsDfD0OwrqV9iW7Xc37r9bxrUYz/W2fQUlpU37vUKOiL2bUJac8uzk70jrEm9Yh5Sv6bDYbqWcK2J+aXdrLLMfezyw5I4/TuUVsOHyaDYdPl3ucs6PRG+3c6rKzFWe+7uZPrbXZbCSdyrUnwDYfyWDHsSyKz+uQ72CB1iE+RgIszI/O4X5EBnjh4HCZ335ZLNAqFiJvMiool78KKdtg7m3Q9g645Xlo0KwKXqHUNS4uLsTExJCQkGBvjm+1WklISGDs2LEXfaybmxuNGzemqKiIL774gnvuuacaIq5ehcVWvt5yDIAhnZuYHI2IiNQmTZs2Zdy4cYwbN87sUOQClBgTkdrNwRHCuxvbLc9D+p6yJFnyRjiy1tiWPguBbcqa9zfqBFoR59KsJUaftzXvwOFzmmk36QrdH4Z2d151fzeLxUKwjxvBPm7c0CKg3H25hcUcTM8xkmWpZQsAHEjPJr/Iyr7UbPalZgPlK1wCvFyNhFlQWaVZi0AvQv3ccbzchFMlnckvYuvRzLJpkUcyOHWBVT0DvFzKGuSH+dOxie+1qTxxcoEeY6DjvbD8ZaOqb+dXsGexsf/Gf2iasVxSfHw8I0eOpEuXLnTr1o2pU6eSk5NDXFwcACNGjKBx48ZMnjwZgLVr15KcnEx0dDTJyclMnDgRq9XKE088YT9ndnY2+/bts98+ePAgiYmJNGjQgPDw8Op9gVdhxZ40TucWEeDlSs/mDc0OR0REqlifPn2Ijo5m6tSpV32u9evX4+npefVBSZVQYkxE6g6LBQJbG9uN/4CsY8ZUy13fGlMv03YZ289vGJVObQYa0y2b3mgkFaRMfiZs/gjWvgsZh419Dk5GIqz7IxDWtVrC8HBx4rpQX64L9S2332q1cSwzz75KpjEl00iYncgqID3b2NYePFXuca5ODjQL8LQ3/W8e5EVkgJE4q0xyqsRqY19qNolHynqD7Uk9g618MRgujg60C/UpXSXS6A3WxN+9antseDaE296ALg/CkvFwYDn88m/Y/D9jld/oP6n/mPyuYcOGkZaWxrPPPktKSgrR0dEsXrzY3pA/KSkJh3O+VMjPz2fChAkcOHAALy8vBg4cyIcffoifn5/9mA0bNnDTTTfZb5/tHTZy5Ejmzp1bLa/rWjg7jfLO6NBaN41bRESuPZvNRklJCU5Olx5DBgYGVkNE5iksLKzViytYbLbzh/G1T1ZWFr6+vmRmZuLjo9W4ROQC8jKMpv27vjGa+Bdml93n6gMtby1t3t+vfq/qd3K/kQxL/F/ZNXL3h5g46PoQ+Nb8VdjO5BfZE2bnJs4OpedSWFKxqf1ZjXzdKqyYGRnoSSNfN07lFJJ4pLQv2JHTbDmSSXZBcYVzNPF3tyfAOoX70S7UB1cnE5NQNptRMbbkGWOaMUBIx9L+Yz3Ni6sG0RiidjD7fcrMK6LrSz9SWGzlm0f/QPvGvpd+kIiIAMaXKAcPHqRZs2a4ubmZHc5lGTVqFO+//365fXPmzCEuLo7vvvuOCRMmsG3bNn744QfCwsKIj49nzZo15OTk0LZtWyZPnky/fv3sjz1/KqXFYmHWrFl8++23LFmyhMaNG/PGG29wxx13XDK2kpIS/vKXv7Bs2TJSUlIIDw/nb3/7G3//+9/LHTd79mzeeOMN9u3bR4MGDbjrrruYPn06ABkZGTz55JMsWrSIzMxMWrRowSuvvMLtt9/OxIkTWbRoEYmJifZzTZ06lalTp3Lo0CH79cnIyKBr167MmDEDV1dXDh48yIcffsi0adPYvXs3np6e3HzzzUydOpWgoLK+z7/99htPPvkkK1euxGazER0dzdy5c0lOTqZv374cOXKkXG/TcePGsXHjRn7++ecLXo+Lfb4ud/ygijERqR/c/aDD3cZWXHBO8/7vIfsEbP/c2BxdoNk5zfu9g82OvOrZbHBwhTFdcs8SoPT7ksA2cP0j0OEecPEwNcTK8HZzJirMj6gwv3L7S6w2jp7OLVddtj/VSJydzCnkeGY+xzPz+WVfernHuTo5UFBcMaHm4eJIxya+9kRYdLgfQd41bLBnsUDrAdC8r9GQf8W/IGUrzB1oVP/d8jz4NzU7SpEab/H24xQWW2kZ5MV1oUqgiohcDZvNRl5RiSnP7e7seFmV+9OmTWPPnj20b9+e559/HjASOgBPPfUUr7/+OpGRkfj7+3PkyBEGDhzISy+9hKurKx988AGDBg1i9+7dF20ZMGnSJP71r3/x2muv8dZbb3Hfffdx+PBhGjS4eN9eq9VKkyZN+Oyzz2jYsCG//vorf/nLX2jUqJG9x+c777xDfHw8r7zyCgMGDCAzM5NVq1bZHz9gwADOnDnDRx99RPPmzdmxYweOjpX7MjchIQEfHx+WLl1q31dUVMQLL7xA69atSU1NJT4+nlGjRvHdd98BkJycTK9evejTpw/Lli3Dx8eHVatWUVxcTK9evYiMjOTDDz/k8ccft5/vf//7H//6178qFVtlKTEmIvWPkyu0vMXYbvu30Yts1zfGdnIf7FtqbN/8n9FLq81Ao3l/QEuzI7+2ivJg66dGQ/3UHWX7W8YaCbHIPkZipY5wdDCa9Uc09OTmNuXvy8gtLFsl85xKs8Mnc+1JsRZBXqWVYP5Eh/nRKtir9kyncnKBG8YaK4f+9BJsnAs7voTdZ/uPxav/mMhFLNhkTKMc0rlx1U6FFhGpB/KKSmj37BJTnnvH87F4uFw6DeLr64uLiwseHh726qVdu3YB8Pzzz3PLLbfYj23QoAFRUVH22y+88AILFy7kq6++uujiNaNGjWL48OEAvPzyy7z55pusW7eO/v37XzQ2Z2dnJk2aZL/drFkzVq9ezaeffmpPjL344ov84x//KFdF1rWr0Qrlxx9/ZN26dezcuZNWrVoBXNFq0p6enrz33nvlplA+8MAD9p8jIyN588036dq1K9nZ2Xh5eTFjxgx8fX355JNPcHY2+hSfjQHgwQcfZM6cOfbE2Ndff01+fn6VL+qjxJiI1G8ODka/rLCucMskSDu3ef8GOLrO2H6cCAGtyla4DO1ce5v3Zx2D9e/BhjmQV9qDy9kTOt0H3f4KAS3Mjc8Efh4uxES4EBPhX25/YbGV5Iw8Gni44Oth/kqXV80zAG7/d1n/sYMr4ZcpxtTZvs9C1J9q7+dapIocPZ1r71d4Z3TNn04uIiJVq0uXLuVuZ2dnM3HiRL799luOHz9OcXExeXl5JCUlXfQ8HTt2tP/s6emJj48PqamplxXDjBkzmD17NklJSeTl5VFYWEh0dDQAqampHDt2jL59+17wsYmJiTRp0qRcQupKdOjQoUJfsY0bNzJx4kS2bNnC6dOnsVqNL5iTkpJo164diYmJ3Hjjjfak2PlGjRrFhAkTWLNmDddffz1z587lnnvuqfKFC5QYExE5V2ArCIw3KmiyjhvN+3d/BwdWGCte/rLHaGTuFVJaSXYbNO1VO5r3H90Ia96GHYvAWtofyy/cSIZ1ut+YbirluJQ2669zQtrDiK+Mz/aSZ+D0QfhyjDHdsv8rEHGD2RGK1BhfJh4D4PrIBjT2czc5GhGR2s/d2ZEdz8ea9txX6/wkzWOPPcbSpUt5/fXXadGiBe7u7tx9990UFlZcnfxc5yeHLBaLPZF0MZ988gmPPfYYb7zxBj169MDb25vXXnuNtWvXAuDufvG/VZe638HBgfNb0RcVFVU47vzrkJOTQ2xsLLGxsfzvf/8jMDCQpKQkYmNj7dfiUs8dFBTEoEGDmDNnDs2aNeP7779n+fLlF33MtaDEmIjI7/FpBF0fNLb8zNLm/d/Cnh8gOwU2zDY2Vx9jWmab26DFLTWreX9JEez8CtbMNCrfzoroaUyXbD1QKxTWVxZL2YITZ/uPHd8CcwbAdUOg3yTwjzA7ShFT2Ww2FpauRjmkk6rFRESuBYvFclnTGc3m4uJCScmle6GtWrWKUaNGMWTIEMCoIDvbpL4qrFq1ihtuuIG//e1v9n379++3/+zt7U3Tpk1JSEgotyr0WR07duTo0aPs2bPnglVjgYGBpKSkYLPZ7O0Dzm3E/3t27drFyZMneeWVVwgLCwOMlanPf+7333+foqKi360ae+ihhxg+fDhNmjShefPm9OxZ9QtGab6EiMjlcPOF9nfB3bPhif1w/xfQ5QGjcqwgC7Z/AZ8/AP+KhA+Hwvr/GhVnZsk9ZVS2TYsy4jq6zlhYIOpP8NeVEPcdtB2kpJgYPfdueBQe3QQxo8DiAL8thOldIeEFKMi+5ClE6qrfjmWxLzUbVycHBnRoZHY4IiJSjZo2bcratWs5dOgQ6enpv1vN1bJlSxYsWEBiYiJbtmzhT3/602VVfl2pli1bsmHDBpYsWcKePXv45z//yfr168sdM3HiRN544w3efPNN9u7dy6ZNm3jrrbcA6N27N7169eKuu+5i6dKlHDx4kO+//57FixcD0KdPH9LS0vjXv/7F/v37mTFjBt9///0l4woPD8fFxYW33nqLAwcO8NVXX/HCCy+UO2bs2LFkZWVx7733smHDBvbu3cuHH37I7t277cfExsbi4+PDiy++SFxc3NVersuixJiISGU5uRpVNrf/G+J3wkMJ8If/M3qQWYtgfwJ8Gw9T2sCsvvDzFKN3WXVI3QVfj4Mp7Yy+aFnJ4BkIfcbD//0GQ96BRlGXOovUR16BMGiakThteiOUFMDPr8NbMZA4D6pwgCdSU51tut+vXTA+bnWgz6CIiFy2xx57DEdHR9q1a2efFnghU6ZMwd/fnxtuuIFBgwYRGxtL586dqyyuv/71rwwdOpRhw4bRvXt3Tp48Wa56DGDkyJFMnTqVt99+m+uuu47bb7+dvXv32u//4osv6Nq1K8OHD6ddu3Y88cQT9uq4tm3b8vbbbzNjxgyioqJYt24djz322CXjCgwMZO7cuXz22We0a9eOV155hddff73cMQ0bNmTZsmVkZ2fTu3dvYmJimDVrVrnqMQcHB0aNGkVJSQkjRoy4mkt12Sy28yeP1kJZWVn4+vqSmZmJj08NmsIkIvVP+l5juuWub8tPXQRo2LKseX/jmGvX5NxqNaZ5rn0H9i8r2x/SAa7/m1Hp5uR6bZ5L6gebzfgM/zDB6D8GENrJ6D8Wfr25sV1jGkPUDma8T8UlVq6fvIz07ALeG9GFfu2Cq+V5RUTqmvz8fA4ePEizZs1wc3MzOxypBR588EHS0tL46quvLnnsxT5flzt+qPkTe0VEapOAlvCHccZ2JgV2f28kGA6ugJN7YdVUY/MKhtYDjCRZs15XlrgqyIYtH8PamXByn7HP4mAk37o/YjRQL+0LIFIpFgu0vd3onbd2Jqx4DY5thtmxcN1QYwVXv3CzoxSpUr/sSyc9uwB/D2d6tw40OxwREZE6LzMzk23btjFv3rzLSopdK0qMiYhUFe8Q6BJnbPlZZc379/4A2Sdg41xjc/GGlv2MJFnLW4x+ZheTkWQ0S9/4ARRkGvtcfaDzCOg2GvybVvELk3rDyRV6/h2ihsOyF2HTB/DbAmM1yxsehZ7jwNXL7ChFqsTZpvuDokJxdlT3ERERqR4PP/wwH3300QXvu//++5k5c2Y1R1R97rzzTtatW8fDDz/MLbfcUm3Pq6mUIiLVrbgQDv9SNuXyzDlN+h2codmNRtVX64HgE2rst9kgaQ2seRt2fQO20n5PDZpD94cheji4elf/a5H65fhWWPI0HPrZuO3dCPo+Bx2HXbupwdVMY4jaobrfp+yCYrq8uJT8IisL/3YDncL9q/w5RUTqKk2lrJzU1FSysrIueJ+Pjw9BQUHVHFHNpqmUIiK1kZMLNL/Z2Aa8Bsc3lyXJ0nYZfcL2L4Nv/wGhnSGyj3H7eGLZOSL7GP3DWtxSaxMSUgs16ggjv4adXxv9xzIOw6KHjQrG/q9AeHezIxS5JpZsTyG/yEqzAE+iw/zMDkdEROqRoKAgJb+qmRJjIiJmcnAwGvE3joG+z0L6PthdmiQ7sg6ObTI2ACc3ozKn+8MQ3M7cuKX+slig3R3Q8lZjwYeVbxif0dm3Qvu7od9E8AszO0qRq7Io0ZhGOTi6MRb1ahQREanTlBgTEalJAlpAwN+Nvk5nTsCe7+HQKghqA51HgWdDsyMUMTi7wR/+D6L+BMtegM0fwfbPjaRuz/9nfIZdPM2OUqTSTmTls2pfOgBDOjU2ORoRERGpapp/IyJSU3kHQ8wouGsW3PgPJcWkZvIOhjunw19XQERPKM6DFa/CW11gy3ywWs2OUKRSvkxMxmqDmAh/wht6mB2OiIiIVDElxkREROTqNYqCUd/CPR+AXzicOQYL/wL/vQWOrDc7OpHLtnDzMUDVYiIiIvWFEmMiIiJybVgs0O5OGLPeWK3SxQuSN8B/+8EXD0HmUbMjFLmoXSlZ7DyehbOjhds6NDI7HBEREakGV5QYmzFjBk2bNsXNzY3u3buzbt26ix7/2Wef0aZNG9zc3OjQoQPfffdduftHjRqFxWIpt/Xv3/9KQhMRERGzObvBjfHw6EbodD9ggW2fGdMrf5oMhblmRyhyQQs3G033b2odhL+ni8nRiIiISHWodGJs/vz5xMfH89xzz7Fp0yaioqKIjY0lNTX1gsf/+uuvDB8+nAcffJDNmzczePBgBg8ezPbt28sd179/f44fP27fPv744yt7RSIiIlIzeIfAnTPgLz9B+A2l/cdegeldYOun6j8mNUqJ1caXpdMoh3bWNEoRkfquT58+jBs37pqdb9SoUQwePPianU+unUonxqZMmcLo0aOJi4ujXbt2zJw5Ew8PD2bPnn3B46dNm0b//v15/PHHadu2LS+88AKdO3dm+vTp5Y5zdXUlJCTEvvn7+1/ZKxIREZGaJbQTxH0Hf5wLvuGQlQwLRhv9x45uMDs6EQDWHDhJSlY+Pm5O3NQmyOxwREREapzCwkKzQ6gSlUqMFRYWsnHjRvr161d2AgcH+vXrx+rVqy/4mNWrV5c7HiA2NrbC8cuXLycoKIjWrVvzyCOPcPLkyd+No6CggKysrHKbiIiI1GAWC1w3BMauh5v/Cc6eRv+x9/rCgr9AZrLZEUo9d3Ya5W0dQ3F1cjQ5GhERMdOoUaNYsWIF06ZNs7d7OnToENu3b2fAgAF4eXkRHBzMn//8Z9LT0+2P+/zzz+nQoQPu7u40bNiQfv36kZOTw8SJE3n//ff58ssv7edbvnz5JeN48sknadWqFR4eHkRGRvLPf/6ToqKicsd8/fXXdO3aFTc3NwICAhgyZIj9voKCAp588knCwsJwdXWlRYsW/Pe//wVg7ty5+Pn5lTvXokWLsFgs9tsTJ04kOjqa9957j2bNmuHm5gbA4sWL+cMf/oCfnx8NGzbk9ttvZ//+/eXOdfToUYYPH06DBg3w9PSkS5curF27lkOHDuHg4MCGDeW/HJ06dSoRERFYTZhRUKnEWHp6OiUlJQQHB5fbHxwcTEpKygUfk5KScsnj+/fvzwcffEBCQgKvvvoqK1asYMCAAZSUlFzwnJMnT8bX19e+hYWFVeZliIiIiFmc3aDXY/D/NkH0fca+rfON6ZXLX1X/MTFFXmEJ3287Dmg1ShGRKmezQWGOOZvNdlkhTps2jR49ejB69Gh7uydvb29uvvlmOnXqxIYNG1i8eDEnTpzgnnvuAeD48eMMHz6cBx54gJ07d7J8+XKGDh2KzWbjscce45577inXQuqGG264ZBze3t7MnTuXHTt2MG3aNGbNmsW///1v+/3ffvstQ4YMYeDAgWzevJmEhAS6detmv3/EiBF8/PHHvPnmm+zcuZN3330XLy+vSr1d+/bt44svvmDBggUkJiYCkJOTQ3x8PBs2bCAhIQEHBweGDBliT2plZ2fTu3dvkpOT+eqrr9iyZQtPPPEEVquVpk2b0q9fP+bMmVPueebMmcOoUaNwcKj+NSKdqv0ZL+Dee++1/9yhQwc6duxI8+bNWb58OX379q1w/Pjx44mPj7ffzsrKUnJMRESkNvEOgcFvQ7fR8P1TcGQNLH8ZNr0P/SZBh7uNKjORarB05wlyCkto4u9Olwi18xARqVJFufByqDnP/fQxcPG85GG+vr64uLjg4eFBSEgIAC+++CKdOnXi5Zdfth83e/ZswsLC2LNnD9nZ2RQXFzN06FAiIiIAI79xlru7OwUFBfbzXY4JEybYf27atCmPPfYYn3zyCU888QQAL730Evfeey+TJk2yHxcVFQXAnj17+PTTT1m6dKl9Fl9kZORlP/dZhYWFfPDBBwQGBtr33XXXXeWOmT17NoGBgezYsYP27dszb9480tLSWL9+PQ0aNACgRYsW9uMfeughHn74YaZMmYKrqyubNm1i27ZtfPnll5WO71qoVCouICAAR0dHTpw4UW7/iRMnfvfNDQkJqdTxYLxZAQEB7Nu374L3u7q64uPjU24TERGRWii0EzywGO6eA75hpf3HHoL/3gpHN5odndQTCzcdBYxqMQcHJWRFRKSiLVu28NNPP+Hl5WXf2rRpA8D+/fuJioqib9++dOjQgT/+8Y/MmjWL06dPX9Vzzp8/n549exISEoKXlxcTJkwgKSnJfn9iYuIFi4nO3ufo6Ejv3r2vKoaIiIhySTGAvXv3Mnz4cCIjI/Hx8aFp06YA9tgSExPp1KmTPSl2vsGDB+Po6MjChQsBY1rnTTfdZD9PdatUxZiLiwsxMTEkJCTYV1OwWq0kJCQwduzYCz6mR48eJCQklFvNYenSpfTo0eN3n+fo0aOcPHmSRo0aVSY8ERERqY0sFmg/FFoPgNXT4ed/w9F18N7N0PFe6Pcc+Jj0zbLUeWlnCli51+gPM1jTKEVEqp6zh1G5ZdZzX6Hs7GwGDRrEq6++WuG+Ro0a4ejoyNKlS/n111/54YcfeOutt3jmmWdYu3YtzZo1q/TzrV69mvvuu49JkyYRGxuLr68vn3zyCW+88Yb9GHd39999/MXuA6NfvO28qaXn9y8D8PSsWGE3aNAgIiIimDVrFqGhoVitVtq3b29vzn+p53ZxcWHEiBHMmTOHoUOHMm/ePKZNm3bRx1SlSk/ejI+PZ9asWbz//vvs3LmTRx55hJycHOLi4gBjDuv48ePtx//9739n8eLFvPHGG+zatYuJEyeyYcMGeyItOzubxx9/nDVr1nDo0CESEhK48847adGiBbGxsdfoZYqIiEiN5+wOvR6HRzdC1J+MfVs/gbe6wJkTF3+syBX6ZusxSqw2opr40jywcn1XRETkClgsxnRGM7ZKtGlwcXEp1/e8c+fO/PbbbzRt2pQWLVqU284mjywWCz179mTSpEls3rwZFxcXe1XU+ee7lF9//ZWIiAieeeYZunTpQsuWLTl8+HC5Yzp27EhCQsIFH9+hQwesVisrVqy44P2BgYGcOXOGnJwc+76zPcQu5uTJk+zevZsJEybQt29f2rZtW6EyrmPHjiQmJnLq1KnfPc9DDz3Ejz/+yNtvv22fgmqWSifGhg0bxuuvv86zzz5LdHQ0iYmJLF682N5gPykpiePHj9uPv+GGG5g3bx7/+c9/iIqK4vPPP2fRokW0b98eAEdHR7Zu3codd9xBq1atePDBB4mJieHnn3/G1dX1Gr1MERERqTV8GsGQd2D0MgjrblSSeQdf+nEiV8DT1YmmDT1ULSYiIuU0bdrUvopieno6Y8aM4dSpUwwfPpz169ezf/9+lixZQlxcHCUlJaxdu5aXX36ZDRs2kJSUxIIFC0hLS6Nt27b2823dupXdu3eTnp5+weqsc7Vs2ZKkpCQ++eQT9u/fz5tvvmlPsp313HPP8fHHH/Pcc8+xc+dOtm3bZq9oa9q0KSNHjuSBBx5g0aJFHDx4kOXLl/Ppp58C0L17dzw8PHj66afZv38/8+bNY+7cuZe8Lv7+/jRs2JD//Oc/7Nu3j2XLlpXrAQ8wfPhwQkJCGDx4MKtWreLAgQN88cUXrF692n5M27Ztuf7663nyyScZPnz4JavMqpStDsjMzLQBtszMTLNDERERkWvJarXZ8s9U2ek1hqgdqvp9slqttsLikio5t4hIfZeXl2fbsWOHLS8vz+xQKmX37t2266+/3ubu7m4DbAcPHrTt2bPHNmTIEJufn5/N3d3d1qZNG9u4ceNsVqvVtmPHDltsbKwtMDDQ5urqamvVqpXtrbfesp8vNTXVdsstt9i8vLxsgO2nn366ZAyPP/64rWHDhjYvLy/bsGHDbP/+979tvr6+5Y754osvbNHR0TYXFxdbQECAbejQofb78vLybP/3f/9na9Sokc3FxcXWokUL2+zZs+33L1y40NaiRQubu7u77fbbb7f95z//sZ2bJnruuedsUVFRFeJaunSprW3btjZXV1dbx44dbcuXL7cBtoULF9qPOXTokO2uu+6y+fj42Dw8PGxdunSxrV27ttx5/vvf/9oA27p16y55LX7PxT5flzt+sNhsl7leaQ2WlZWFr68vmZmZasQvIiIil01jiNpB75OISO2Vn5/PwYMHadasGW5ubmaHIzXICy+8wGeffcbWrVuv+BwX+3xd7vih0lMpRURERERERERErkR2djbbt29n+vTpPProo2aHo8SYiIiIiIiIiEh1evnll/Hy8rrgNmDAALPDq1Jjx44lJiaGPn368MADD5gdjhJjIiIiIlJmxowZNG3aFDc3N7p37866det+99iioiKef/55mjdvjpubG1FRUSxevPiqzikiIlIfPPzwwyQmJl5we++998wOr0rNnTuXgoIC5s+fj6Ojo9nh4GR2ACIiIiJSM8yfP5/4+HhmzpxJ9+7dmTp1KrGxsezevZugoKAKx0+YMIGPPvqIWbNm0aZNG5YsWcKQIUP49ddf6dSp0xWdU0REpD5o0KABDRo0MDsMQRVjIiIiIlJqypQpjB49mri4ONq1a8fMmTPx8PBg9uzZFzz+ww8/5Omnn2bgwIFERkbyyCOPMHDgQN54440rPqeIiIhIdVJiTEREREQoLCxk48aN9OvXz77PwcGBfv36sXr16gs+pqCgoMIKUO7u7vzyyy9XfE4REambbDab2SFIHWS1Wq/6HJpKKSIiIiKkp6dTUlJCcHBwuf3BwcHs2rXrgo+JjY1lypQp9OrVi+bNm5OQkMCCBQsoKSm54nOCkXArKCiw387KyrrSlyUiIiZzdnbGYrGQlpZGYGAgFovF7JCkDrDZbBQWFpKWloaDgwMuLi5XfC4lxkRERETkikybNo3Ro0fTpk0bLBYLzZs3Jy4u7qqnSU6ePJlJkyZdoyhFRMRMjo6ONGnShKNHj3Lo0CGzw5E6xsPDg/DwcBwcrnxCpBJjIiIiIkJAQACOjo6cOHGi3P4TJ04QEhJywccEBgayaNEi8vPzOXnyJKGhoTz11FNERkZe8TkBxo8fT3x8vP12VlYWYWFhV/rSRETEZF5eXrRs2ZKioiKzQ5E6xNHREScnp6uuQlRiTERERERwcXEhJiaGhIQEBg8eDBh9OxISEhg7duxFH+vm5kbjxo0pKiriiy++4J577rmqc7q6uuLq6npNXpeIiNQMjo6OODo6mh2GSAVKjImIiIgIAPHx8YwcOZIuXbrQrVs3pk6dSk5ODnFxcQCMGDGCxo0bM3nyZADWrl1LcnIy0dHRJCcnM3HiRKxWK0888cRln1NERETETEqMiYiIiAgAw4YNIy0tjWeffZaUlBSio6NZvHixvXl+UlJSuR4e+fn5TJgwgQMHDuDl5cXAgQP58MMP8fPzu+xzioiIiJjJYqsDa6ZmZWXh6+tLZmYmPj4+ZocjIiIitYTGELWD3icRERGprMsdP9SJirGzuT0t5S0iIiKVcXbsUAe+J6zTNNYTERGRyrrccV6dSIydOXMGQKsViYiIyBU5c+YMvr6+Zochv0NjPREREblSlxrn1YmplFarlWPHjuHt7X3Vy3ReyNklwo8cOaLyfZPoPTCXrr+5dP3Npetvrqq+/jabjTNnzhAaGlqud5bULBrr1W26/ubS9TeXrr+5dP3NVVPGeXWiYszBwYEmTZpU+fP4+PjofxaT6T0wl66/uXT9zaXrb66qvP6qFKv5NNarH3T9zaXrby5df3Pp+pvL7HGevhoVEREREREREZF6SYkxERERERERERGpl5QYuwyurq4899xzuLq6mh1KvaX3wFy6/ubS9TeXrr+5dP2lOuhzZi5df3Pp+ptL199cuv7mqinXv0403xcREREREREREaksVYyJiIiIiIiIiEi9pMSYiIiIiIiIiIjUS0qMiYiIiIiIiIhIvaTEmIiIiIiIiIiI1EtKjF2GGTNm0LRpU9zc3OjevTvr1q0zO6R6Y+XKlQwaNIjQ0FAsFguLFi0yO6R6Y/LkyXTt2hVvb2+CgoIYPHgwu3fvNjuseuOdd96hY8eO+Pj44OPjQ48ePfj+++/NDqveeuWVV7BYLIwbN87sUOqNiRMnYrFYym1t2rQxOyypgzTOM4/GeebROM9cGufVLBrnVb+aNs5TYuwS5s+fT3x8PM899xybNm0iKiqK2NhYUlNTzQ6tXsjJySEqKooZM2aYHUq9s2LFCsaMGcOaNWtYunQpRUVF3HrrreTk5JgdWr3QpEkTXnnlFTZu3MiGDRu4+eabufPOO/ntt9/MDq3eWb9+Pe+++y4dO3Y0O5R657rrruP48eP27ZdffjE7JKljNM4zl8Z55tE4z1wa59UcGueZpyaN8yw2m81m2rPXAt27d6dr165Mnz4dAKvVSlhYGI8++ihPPfWUydHVLxaLhYULFzJ48GCzQ6mX0tLSCAoKYsWKFfTq1cvscOqlBg0a8Nprr/Hggw+aHUq9kZ2dTefOnXn77bd58cUXiY6OZurUqWaHVS9MnDiRRYsWkZiYaHYoUodpnFdzaJxnLo3zzKdxXvXTOM88NW2cp4qxiygsLGTjxo3069fPvs/BwYF+/fqxevVqEyMTqX6ZmZmA8UdbqldJSQmffPIJOTk59OjRw+xw6pUxY8Zw2223lfs7INVn7969hIaGEhkZyX333UdSUpLZIUkdonGeSBmN88yjcZ55NM4zV00a5zmZ9sy1QHp6OiUlJQQHB5fbHxwczK5du0yKSqT6Wa1Wxo0bR8+ePWnfvr3Z4dQb27Zto0ePHuTn5+Pl5cXChQtp166d2WHVG5988gmbNm1i/fr1ZodSL3Xv3p25c+fSunVrjh8/zqRJk7jxxhvZvn073t7eZocndYDGeSIGjfPMoXGeuTTOM1dNG+cpMSYilzRmzBi2b9+u/j7VrHXr1iQmJpKZmcnnn3/OyJEjWbFihQZN1eDIkSP8/e9/Z+nSpbi5uZkdTr00YMAA+88dO3ake/fuRERE8Omnn2qaiYjINaRxnjk0zjOPxnnmq2njPCXGLiIgIABHR0dOnDhRbv+JEycICQkxKSqR6jV27Fi++eYbVq5cSZMmTcwOp15xcXGhRYsWAMTExLB+/XqmTZvGu+++a3Jkdd/GjRtJTU2lc+fO9n0lJSWsXLmS6dOnU1BQgKOjo4kR1j9+fn60atWKffv2mR2K1BEa54lonGcmjfPMo3FezWP2OE89xi7CxcWFmJgYEhIS7PusVisJCQma/y11ns1mY+zYsSxcuJBly5bRrFkzs0Oq96xWKwUFBWaHUS/07duXbdu2kZiYaN+6dOnCfffdR2JiogZLJsjOzmb//v00atTI7FCkjtA4T+ozjfNqHo3zqo/GeTWP2eM8VYxdQnx8PCNHjqRLly5069aNqVOnkpOTQ1xcnNmh1QvZ2dnlssYHDx4kMTGRBg0aEB4ebmJkdd+YMWOYN28eX375Jd7e3qSkpADg6+uLu7u7ydHVfePHj2fAgAGEh4dz5swZ5s2bx/Lly1myZInZodUL3t7eFfqseHp60rBhQ/VfqSaPPfYYgwYNIiIigmPHjvHcc8/h6OjI8OHDzQ5N6hCN88ylcZ55NM4zl8Z55tI4z3w1bZynxNglDBs2jLS0NJ599llSUlKIjo5m8eLFFRq1StXYsGEDN910k/12fHw8ACNHjmTu3LkmRVU/vPPOOwD06dOn3P45c+YwatSo6g+onklNTWXEiBEcP34cX19fOnbsyJIlS7jlllvMDk2kWhw9epThw4dz8uRJAgMD+cMf/sCaNWsIDAw0OzSpQzTOM5fGeebROM9cGudJfVfTxnkWm81mM+WZRURERERERERETKQeYyIiIiIiIiIiUi8pMSYiIiIiIiIiIvWSEmMiIiIiIiIiIlIvKTEmIiIiIiIiIiL1khJjIiIiIiIiIiJSLykxJiIiIiIiIiIi9ZISYyIiIiIiIiIiUi8pMSYiIiIiIiIiIvWSEmMiIiIiIiIiIlIvKTEmIiIiIiIiIiL1khJjIiIiIiIiIiJSLykxJiIiIiIiIiIi9dL/B6AiSno4T+9dAAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-07-10 15:24:26.290421: W tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[train] step: 200, loss: 0.3102289140224457, accuracy: 90.08084869384766\n", + "[test] step: 200, loss: 0.13239526748657227, accuracy: 95.52284240722656\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-07-10 15:24:32.398018: W tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[train] step: 400, loss: 0.12522409856319427, accuracy: 96.515625\n", + "[test] step: 400, loss: 0.07021520286798477, accuracy: 97.8465576171875\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-07-10 15:24:38.439548: W tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[train] step: 600, loss: 0.09092658758163452, accuracy: 97.25\n", + "[test] step: 600, loss: 0.08268354833126068, accuracy: 97.30569458007812\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-07-10 15:24:44.516602: W tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[train] step: 800, loss: 0.07523862272500992, accuracy: 97.921875\n", + "[test] step: 800, loss: 0.060881033539772034, accuracy: 98.036865234375\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-07-10 15:24:50.557494: W tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[train] step: 1000, loss: 0.063808374106884, accuracy: 98.09375\n", + "[test] step: 1000, loss: 0.07719086110591888, accuracy: 97.4258804321289\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-07-10 15:24:54.450444: W tensorflow/core/kernels/data/cache_dataset_ops.cc:858] The calling iterator did not fully read the dataset being cached. In order to avoid unexpected truncation of the dataset, the partially cached contents of the dataset will be discarded. This can happen if you have an input pipeline similar to `dataset.cache().take(k).repeat()`. You should use `dataset.take(k).cache().repeat()` instead.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[train] step: 1199, loss: 0.07750937342643738, accuracy: 97.47173309326172\n", + "[test] step: 1199, loss: 0.05415954813361168, accuracy: 98.32732391357422\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-07-10 15:24:56.610632: W tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n", + "2024-07-10 15:24:56.615182: W tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" + ] } ], "source": [ - "from IPython.display import clear_output\n", - "import matplotlib.pyplot as plt\n", - "\n", "metrics_history = {\n", " 'train_loss': [],\n", " 'train_accuracy': [],\n", @@ -369,17 +443,60 @@ " metrics_history[f'test_{metric}'].append(value)\n", " metrics.reset() # Reset the metrics for the next training epoch.\n", "\n", - " clear_output(wait=True)\n", - " # Plot loss and accuracy in subplots\n", - " fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 5))\n", - " ax1.set_title('Loss')\n", - " ax2.set_title('Accuracy')\n", - " for dataset in ('train', 'test'):\n", - " ax1.plot(metrics_history[f'{dataset}_loss'], label=f'{dataset}_loss')\n", - " ax2.plot(metrics_history[f'{dataset}_accuracy'], label=f'{dataset}_accuracy')\n", - " ax1.legend()\n", - " ax2.legend()\n", - " plt.show()" + " print(\n", + " f\"[train] step: {step}, \"\n", + " f\"loss: {metrics_history['train_loss'][-1]}, \"\n", + " f\"accuracy: {metrics_history['train_accuracy'][-1] * 100}\"\n", + " )\n", + " print(\n", + " f\"[test] step: {step}, \"\n", + " f\"loss: {metrics_history['test_loss'][-1]}, \"\n", + " f\"accuracy: {metrics_history['test_accuracy'][-1] * 100}\"\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "23", + "metadata": {}, + "source": [ + "## 7. Visualize the metrics\n", + "\n", + "With Matplotlib, you can create plots for the loss and the accuracy:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "24", + "metadata": { + "outputId": "431a2fcd-44fa-4202-f55a-906555f060ac" + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABMYAAAHDCAYAAADP+BbYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAADGZElEQVR4nOzdd3xV9eHG8c+9N3snZIdA2GGGjcgQ2aCAKDh+WoS2tnW0tdRFq2i1FVep1lq1WhQVFwqCAxRZgiLIXmGvJGSH7H3v+f1xISEFlECSc5M879frvhrOOffmuaSYkyffYTEMw0BERERERERERKSZsZodQERERERERERExAwqxkREREREREREpFlSMSYiIiIiIiIiIs2SijEREREREREREWmWVIyJiIiIiIiIiEizpGJMRERERERERESaJRVjIiIiIiIiIiLSLKkYExERERERERGRZknFmIiIiIiIiIiINEsqxkREREREREREpFlSMSYiLu/NN9/EYrGwefNms6OIiIiIyGn//ve/sVgsDBgwwOwoIiKXTMWYiIiIiIiI1NqCBQuIi4tj06ZNHDp0yOw4IiKXRMWYiIiIiIiI1MrRo0f57rvvmDt3LmFhYSxYsMDsSOdVVFRkdgQRcXEqxkSkSdi2bRvjxo0jICAAPz8/RowYwffff1/jmoqKCv7yl7/QoUMHvLy8aNGiBYMHD2bFihVV16SlpTFjxgxatmyJp6cnUVFRTJo0iWPHjjXwOxIRERFxXQsWLCA4OJhrrrmGKVOmnLcYy83N5Q9/+ANxcXF4enrSsmVLpk2bRlZWVtU1paWlPPbYY3Ts2BEvLy+ioqK4/vrrOXz4MABr1qzBYrGwZs2aGq997NgxLBYLb775ZtWx6dOn4+fnx+HDhxk/fjz+/v7ceuutAKxbt46pU6fSqlUrPD09iY2N5Q9/+AMlJSXn5N63bx833ngjYWFheHt706lTJ/785z8DsHr1aiwWC4sXLz7nee+++y4Wi4UNGzbU+u9TRMzjZnYAEZHLtWfPHoYMGUJAQAAPPPAA7u7uvPrqqwwbNoy1a9dWrXvx2GOPMWfOHH75y1/Sv39/8vPz2bx5M1u3bmXUqFEA3HDDDezZs4ff/va3xMXFkZGRwYoVKzhx4gRxcXEmvksRERER17FgwQKuv/56PDw8uOWWW3j55Zf54Ycf6NevHwCFhYUMGTKExMREfv7zn9O7d2+ysrJYunQpycnJhIaGYrfbufbaa1m5ciU333wzv//97ykoKGDFihXs3r2bdu3a1TpXZWUlY8aMYfDgwTz33HP4+PgAsHDhQoqLi7nzzjtp0aIFmzZt4sUXXyQ5OZmFCxdWPX/nzp0MGTIEd3d3fvWrXxEXF8fhw4f59NNP+dvf/sawYcOIjY1lwYIFTJ48+Zy/k3bt2jFw4MDL+JsVkQZniIi4uDfeeMMAjB9++OG856+77jrDw8PDOHz4cNWxkydPGv7+/sbQoUOrjiUkJBjXXHPNBT/PqVOnDMB49tln6y68iIiISBOzefNmAzBWrFhhGIZhOBwOo2XLlsbvf//7qmtmz55tAMaiRYvOeb7D4TAMwzDmzZtnAMbcuXMveM3q1asNwFi9enWN80ePHjUA44033qg6dvvttxuA8dBDD53zesXFxeccmzNnjmGxWIzjx49XHRs6dKjh7+9f49jZeQzDMGbNmmV4enoaubm5VccyMjIMNzc349FHHz3n84iIa9NUShFp1Ox2O1999RXXXXcdbdu2rToeFRXF//3f/7F+/Xry8/MBCAoKYs+ePRw8ePC8r+Xt7Y2Hhwdr1qzh1KlTDZJfREREpLFZsGABERERXH311QBYLBZuuukm3n//fex2OwAff/wxCQkJ54yqOnP9mWtCQ0P57W9/e8FrLsWdd955zjFvb++qj4uKisjKyuLKK6/EMAy2bdsGQGZmJt988w0///nPadWq1QXzTJs2jbKyMj766KOqYx988AGVlZXcdtttl5xbRMyhYkxEGrXMzEyKi4vp1KnTOec6d+6Mw+EgKSkJgMcff5zc3Fw6duxI9+7duf/++9m5c2fV9Z6enjz99NMsW7aMiIgIhg4dyjPPPENaWlqDvR8RERERV2a323n//fe5+uqrOXr0KIcOHeLQoUMMGDCA9PR0Vq5cCcDhw4fp1q3bj77W4cOH6dSpE25udbfCj5ubGy1btjzn+IkTJ5g+fTohISH4+fkRFhbGVVddBUBeXh4AR44cAfjJ3PHx8fTr16/GumoLFizgiiuuoH379nX1VkSkgagYE5FmY+jQoRw+fJh58+bRrVs3Xn/9dXr37s3rr79edc29997LgQMHmDNnDl5eXjzyyCN07ty56jeJIiIiIs3ZqlWrSE1N5f3336dDhw5VjxtvvBGgznenvNDIsTMj0/6Xp6cnVqv1nGtHjRrF559/zoMPPsgnn3zCihUrqhbudzgctc41bdo01q5dS3JyMocPH+b777/XaDGRRkqL74tIoxYWFoaPjw/79+8/59y+ffuwWq3ExsZWHQsJCWHGjBnMmDGDwsJChg4dymOPPcYvf/nLqmvatWvHH//4R/74xz9y8OBBevbsyd///nfeeeedBnlPIiIiIq5qwYIFhIeH89JLL51zbtGiRSxevJhXXnmFdu3asXv37h99rXbt2rFx40YqKipwd3c/7zXBwcGAc4fLsx0/fvyiM+/atYsDBw4wf/58pk2bVnX87J3JgaplOX4qN8DNN9/MzJkzee+99ygpKcHd3Z2bbrrpojOJiOvQiDERadRsNhujR49myZIlHDt2rOp4eno67777LoMHDyYgIACA7OzsGs/18/Ojffv2lJWVAVBcXExpaWmNa9q1a4e/v3/VNSIiIiLNVUlJCYsWLeLaa69lypQp5zzuueceCgoKWLp0KTfccAM7duxg8eLF57yOYRiAczfwrKws/vWvf13wmtatW2Oz2fjmm29qnP/3v/990bltNluN1zzz8QsvvFDjurCwMIYOHcq8efM4ceLEefOcERoayrhx43jnnXdYsGABY8eOJTQ09KIziYjr0IgxEWk05s2bx/Lly885/thjj7FixQoGDx7MXXfdhZubG6+++iplZWU888wzVdd16dKFYcOG0adPH0JCQti8eTMfffQR99xzDwAHDhxgxIgR3HjjjXTp0gU3NzcWL15Meno6N998c4O9TxERERFXtHTpUgoKCpg4ceJ5z19xxRWEhYWxYMEC3n33XT766COmTp3Kz3/+c/r06UNOTg5Lly7llVdeISEhgWnTpvHWW28xc+ZMNm3axJAhQygqKuLrr7/mrrvuYtKkSQQGBjJ16lRefPFFLBYL7dq147PPPiMjI+Oic8fHx9OuXTvuu+8+UlJSCAgI4OOPPz7vZkv//Oc/GTx4ML179+ZXv/oVbdq04dixY3z++eds3769xrXTpk1jypQpADzxxBMX/xcpIq7FzC0xRUQuxhtvvGEAF3wkJSUZW7duNcaMGWP4+fkZPj4+xtVXX2189913NV7nr3/9q9G/f38jKCjI8Pb2NuLj442//e1vRnl5uWEYhpGVlWXcfffdRnx8vOHr62sEBgYaAwYMMD788EMz3raIiIiIS5kwYYLh5eVlFBUVXfCa6dOnG+7u7kZWVpaRnZ1t3HPPPUZMTIzh4eFhtGzZ0rj99tuNrKysquuLi4uNP//5z0abNm0Md3d3IzIy0pgyZYpx+PDhqmsyMzONG264wfDx8TGCg4ONX//618bu3bsNwHjjjTeqrrv99tsNX1/f8+bau3evMXLkSMPPz88IDQ017rjjDmPHjh3nvIZhGMbu3buNyZMnG0FBQYaXl5fRqVMn45FHHjnnNcvKyozg4GAjMDDQKCkpuci/RRFxNRbD+J8xoSIiIiIiIiLyoyorK4mOjmbChAn897//NTuOiFwirTEmIiIiIiIiUkuffPIJmZmZNRb0F5HGRyPGRERERERERC7Sxo0b2blzJ0888QShoaFs3brV7Egichk0YkxERERERETkIr388svceeedhIeH89Zbb5kdR0Quk0aMiYiIiIiIiIhIs6QRYyIiIiIiIiIi0iypGBMRERERERERkWbJzewAdcHhcHDy5En8/f2xWCxmxxEREZFGwjAMCgoKiI6OxmrV7wtdle71REREpLYu9j6vSRRjJ0+eJDY21uwYIiIi0kglJSXRsmVLs2PIBeheT0RERC7VT93nNYlizN/fH3C+2YCAAJPTiIiISGORn59PbGxs1b2EuCbd64mIiEhtXex9XpMoxs4MqQ8ICNDNkoiIiNSapue5Nt3riYiIyKX6qfs8LaYhIiIiIiIiIiLNkooxERERERERERFpllSMiYiIiIiIiIhIs9Qk1hgTERGpL3a7nYqKCrNjyCVyd3fHZrOZHUNEREREXJSKMRERkfMwDIO0tDRyc3PNjiKXKSgoiMjISC2wLyIiIiLnUDEmIiJyHmdKsfDwcHx8fFSqNEKGYVBcXExGRgYAUVFRJicSEREREVejYkxEROR/2O32qlKsRYsWZseRy+Dt7Q1ARkYG4eHhmlYpIiIiIjVo8X0REZH/cWZNMR8fH5OTSF0483XUWnEiIiIi8r9UjImIiFyApk82Dfo6ioiIiMiFqBgTEREREREREZFmScWYiIiInFdcXBzPP/98nbzWmjVrsFgs2uVTRERERFyKFt8XERFpQoYNG0bPnj3rpND64Ycf8PX1vfxQIiIiIiIuSsWYiIhIM2IYBna7HTe3n74FCAsLa4BEIiIiIiLm0VTKi1BaYWfpjpOczC0xO4qIiMgFTZ8+nbVr1/LCCy9gsViwWCy8+eabWCwWli1bRp8+ffD09GT9+vUcPnyYSZMmERERgZ+fH/369ePrr7+u8Xr/O5XSYrHw+uuvM3nyZHx8fOjQoQNLly695Lwff/wxXbt2xdPTk7i4OP7+97/XOP/vf/+bDh064OXlRUREBFOmTKk699FHH9G9e3e8vb1p0aIFI0eOpKio6JKziIiIiEgDy0iExE/NTqERYxfjd+9t46u96fx+RAf+MKqj2XFERKSBGYZBSYXdlM/t7W676F0VX3jhBQ4cOEC3bt14/PHHAdizZw8ADz30EM899xxt27YlODiYpKQkxo8fz9/+9jc8PT156623mDBhAvv376dVq1YX/Bx/+ctfeOaZZ3j22Wd58cUXufXWWzl+/DghISG1el9btmzhxhtv5LHHHuOmm27iu+++46677qJFixZMnz6dzZs387vf/Y63336bK6+8kpycHNatWwdAamoqt9xyC8888wyTJ0+moKCAdevWYRhGrTKIiIiISAOrKIE9n8CWNyHpe/AKhPYjwd3btEgqxi7CNT2i+GpvOh9tSeb3IzpgtWrbdxGR5qSkwk6X2V+a8rn3Pj4GH4+L+3YdGBiIh4cHPj4+REZGArBv3z4AHn/8cUaNGlV1bUhICAkJCVV/fuKJJ1i8eDFLly7lnnvuueDnmD59OrfccgsATz75JP/85z/ZtGkTY8eOrdX7mjt3LiNGjOCRRx4BoGPHjuzdu5dnn32W6dOnc+LECXx9fbn22mvx9/endevW9OrVC3AWY5WVlVx//fW0bt0agO7du9fq84uIiIhIA0rfA1vmw873oTTPecxig7ghUHLK1GJMUykvwpiukfh7uZGSW8L3R7LNjiMiIlJrffv2rfHnwsJC7rvvPjp37kxQUBB+fn4kJiZy4sSJH32dHj16VH3s6+tLQEAAGRkZtc6TmJjIoEGDahwbNGgQBw8exG63M2rUKFq3bk3btm352c9+xoIFCyguLgYgISGBESNG0L17d6ZOncprr73GqVOnap1BREREROpReTFsWwCvj4KXr4RNrzpLsaBWMPwRmLkXbl4AAdGmxtSIsYvg5W5jQkI07248wcItyVzZPtTsSCIi0oC83W3sfXyMaZ+7Lvzv7pL33XcfK1as4LnnnqN9+/Z4e3szZcoUysvLf/R13N3da/zZYrHgcDjqJOPZ/P392bp1K2vWrOGrr75i9uzZPPbYY/zwww8EBQWxYsUKvvvuO7766itefPFF/vznP7Nx40batGlT51lEREREpBbSdjmnSu78EMryncesbtBpPPSZDm2vBqvrjNNSMXaRbuwby7sbT/DFrlT+MqkrAV7uP/0kERFpEiwWy0VPZzSbh4cHdvtPr4f27bffMn36dCZPngw4R5AdO3asntNV69y5M99+++05mTp27IjN5iwD3dzcGDlyJCNHjuTRRx8lKCiIVatWcf3112OxWBg0aBCDBg1i9uzZtG7dmsWLFzNz5swGew8iIiIiclpZIexZ5CzEUrZUHw+Oc5ZhPW8Fv3CTwv24xnGX7wISWgbSIdyPgxmFfLYjlf8bcOGFiUVERMwSFxfHxo0bOXbsGH5+fhcczdWhQwcWLVrEhAkTsFgsPPLII/Uy8utC/vjHP9KvXz+eeOIJbrrpJjZs2MC//vUv/v3vfwPw2WefceTIEYYOHUpwcDBffPEFDoeDTp06sXHjRlauXMno0aMJDw9n48aNZGZm0rlz5wbLLyIiIiJA6o7To8MWQnmB85jVHTpf6yzE4oa61Oiw83HtdC7EYrEwtW9LABZuSTI5jYiIyPndd9992Gw2unTpQlhY2AXXDJs7dy7BwcFceeWVTJgwgTFjxtC7d+8Gy9m7d28+/PBD3n//fbp168bs2bN5/PHHmT59OgBBQUEsWrSI4cOH07lzZ1555RXee+89unbtSkBAAN988w3jx4+nY8eOPPzww/z9739n3LhxDZZfREREpNkqK3CWYf8ZBq8Ohc3znKVYSFsY9TjMTISpb0LbYS5figFYjCawt3l+fj6BgYHk5eUREBBQb58ns6CMK+asxO4w+HrmUNqH+9fb5xIREfOUlpZy9OhR2rRpg5eXl9lx5DL92Nezoe4h5PLo6yQiIuICTm5zFmK7PoLyQucxqzt0mXh6dNgQsFjMTFjDxd4/aCplLYT5e3J1p3C+Tkxn4eZkZo3XlA0RERERERERaaJK82H3R85CLHVH9fEW7Z1lWMIt4Nu4NyhUMVZLU/u25OvEdBZtS+H+MZ1ws7n+sEAREZH69pvf/IZ33nnnvOduu+02XnnllQZOJCIiIlJTWaWd3OIKThWXc6qogtziciICvegRE6if7c9mGHBy6+nRYR9DRZHzuM0DukxyFmKtB7nU6LDLcUnF2EsvvcSzzz5LWloaCQkJvPjii/Tv3/+81y5atIgnn3ySQ4cOUVFRQYcOHfjjH//Iz372s6prDMPg0Ucf5bXXXiM3N5dBgwbx8ssv06FDh0t7V/VoeHw4LXw9yCwoY+2BTEZ0jjA7koiIiOkef/xx7rvvvvOe09S3xqU293kVFRXMmTOH+fPnk5KSQqdOnXj66acZO3Zs1TV2u53HHnuMd955h7S0NKKjo5k+fToPP/wwliZyQy0iIg2vtMJOTlF5Vcl1qvg8HxdXcKrqmnKKys+/c3eAlxuD2ocytGMYQzqE0jLYp4HfjYsozYOdH8KW+ZC+q/p4aMfq0WE+IabFqy+1LsY++OADZs6cySuvvMKAAQN4/vnnGTNmDPv37yc8/NytN0NCQvjzn/9MfHw8Hh4efPbZZ8yYMYPw8HDGjBkDwDPPPMM///lP5s+fT5s2bXjkkUcYM2YMe/fudbm1XdxtVib3iuH19UdZuDlZxZiIiAgQHh5+3vsAaVxqe5/38MMP88477/Daa68RHx/Pl19+yeTJk/nuu+/o1asXAE8//TQvv/wy8+fPp2vXrmzevJkZM2YQGBjI7373u4Z+iyIi4mIMw6Co3F5dYBU7R3I5S6+ziq3/Kb1KKy5tN22b1UKQtztBPu4EertzOLOIvJIKlu1OY9nuNADahvpWlWRXtG2Br2cTnmxnGJC82Tk6bM8iqCh2Hrd5QtfJzkKs1RVNZnTY+dR68f0BAwbQr18//vWvfwHgcDiIjY3lt7/9LQ899NBFvUbv3r255ppreOKJJzAMg+joaP74xz9W/aY5Ly+PiIgI3nzzTW6++eaffL2GXpB1f1oBY57/BjerhY1/GkELP896/5wiItJwtPh+06LF9y9ebe/zoqOj+fOf/8zdd99ddeyGG27A29u7amrttddeS0REBP/9738veM1P0ddJRJoUhx2OfweJn8KhFeAd7Ny9r+3VEDsA3DzMTnjJDMMgv7SyqtjKLa6oHtX1P6XXmXO5xRWU2y+t5HK3WQjy8SDEx4MgH3dCfD0I8vEg+KyPQ3zdq64J9vHA38sNq7W65LE7DHYm57LuYBbfHMhkW1IudodR43P0aR3MkA5hDO0QRtfogBrPb7RKck+PDnsTMvZUHw+Lhz4zoMeNjX50WL0svl9eXs6WLVuYNWtW1TGr1crIkSPZsGHDTz7fMAxWrVrF/v37efrppwE4evQoaWlpjBw5suq6wMBABgwYwIYNGy6qGGtonSL96dEykJ3JeXyy/SS/GNzG7EgiIiIil+VS7vPKysrOKRu9vb1Zv3591Z+vvPJK/vOf/3DgwAE6duzIjh07WL9+PXPnzq2fNyIi4ooqy+HoN5C4FPZ9DsVZNc+nbIF1fwd3X4gb5CzJ2l3tLClMGqljdxjkl1SQU1xO7unRWmc+zin6n4Lr9PHc4goqHbUae1PF081ao9gK9j1dcPmcKbiqy6/g00WYn6fbZU/Lt1kt9GoVTK9WwfxuRAfySyvYcDibbw5k8s3BTJJySvj+SA7fH8nh2S/3E+LrweD2oQzp4Jx6GRHQiH6JahiQtOn06LDFUFniPO7mBV2vd44Oi+3fpEeHnU+tirGsrCzsdjsRETWnD0ZERLBv374LPi8vL4+YmBjKysqw2Wz8+9//ZtSoUQCkpaVVvcb/vuaZc/+rrKyMsrKyqj/n5+fX5m3Uial9WrIzOY+Fm5P4+aA4rZEhIiIijdql3OeNGTOGuXPnMnToUNq1a8fKlStZtGgRdnv1Gi4PPfQQ+fn5xMfHY7PZsNvt/O1vf+PWW2+9YBZXuNcTEblsFSVwaKVzZNj+ZVCWV33OKwjir3E+SvPg8Go4shqKMuHgV84HgH9U9WiytsPA/9KW8qm0O8gtOTMt8cxIrfLThVZFjWmMZz7OLamgdvPLqvl42Aj28SDY1935v2cVW87jp8uv0x+H+Hjg7WG7tE9WxwK83BnTNZIxXSMBOJ5ddLoky2LD4WxyispZuuMkS3ecBKBThD9DO4YypEMY/duE4OXuGu+jhpJTsOMDZyGWmVh9PLyrswzrMdU5crGZapCJsv7+/mzfvp3CwkJWrlzJzJkzadu2LcOGDbuk15szZw5/+ctf6jZkLU1MiOGJzxPZl1bA7pR8urcMNDWPiIiISEN74YUXuOOOO4iPj8disdCuXTtmzJjBvHnzqq758MMPWbBgAe+++y5du3Zl+/bt3HvvvURHR3P77bef93Vd4V5PROSSlBXAgS+dZdjBFdW7+QH4hkPna6HzRIgbDDb36nM9/w8cDueUtjMl2fHvoCAVdrznfACEd6WyzVUUxAwlI7g3OeVuZ62/VbPYyjk9bfFUUTn5pZWX/Jb8vdzOLbNqjOqqWYAF+bi7Zjl0iVq38OVnA3352cA4KuwOtp3IZd3BTL45kMnOlDz2pxewP72A19YdxdPNSv82IQztEMaQjqF0ivA3bxCNYcCJ751l2N5PoLLUedzNG7rd4CzEWvZtdqPDzqdWxVhoaCg2m4309PQax9PT04mMjLzg86xWK+3btwegZ8+eJCYmMmfOHIYNG1b1vPT0dKKiomq8Zs+ePc/7erNmzWLmzJlVf87Pzyc2NrY2b+WyBfo4W+RPd5xk4ZYkFWMiIiLSqF3KfV5YWBiffPIJpaWlZGdnEx0dzUMPPUTbtm2rrrn//vt56KGHqpbH6N69O8ePH2fOnDkXLMZc4V5PROSiFec4R4QlfgqHV4G9esQrgbHQeYKzDIvtD9aahVFZpZ3swrMXlm/BKetETkWNo8C/gODsrbTO20R80Wba2w9Dxh7cMvYQzL/xMdzIcnRil6M76xzd2GPEYWC9YEyLBQK93atLrbPLLl+P8xx3llzutgu/ZnPjbnMWX/3bhPDH0Z04VVTOt4eda5OtO5hFal4p6w5mse5gFnwB4f6ezrXJOoYyuH1ow6xPXpwDO953FmJZ+6uPR3Q7PTrsRvBSf3G2WhVjHh4e9OnTh5UrV3LdddcBzkVZV65cyT333HPRr+NwOKqGx7dp04bIyEhWrlxZVYTl5+ezceNG7rzzzvM+39PTE09P8xe8n9qnJZ/uOMmS7Sf50/jOTaoVFxERuRTHjh2jTZs2bNu27YK/4BLXdDn3eV5eXsTExFBRUcHHH3/MjTfeWHWuuLgYq7XmD1U2mw2H48ILLbvKvZ6IyAUVpMP+z2HvUji2DhxnjcgKaQddJjrLsOhe54zIScopZvX+DFYmZrDhSDbllT+28HwUMAmYRDD5DLLuYbB1F0Nsu4mxZDHItodBtj08CBTZAjke2JfMsCspiB6CZ1hcjdIr0NsdW1NYNN6FBPt6cG2PaK7tEY1hGBzOLGTtgSzWHczk+yPZZBSU8fHWZD7emgxAt5iAqkX8+7QOxsOtjkpHw4Dj354eHba0upx19zk9OmwGxPTW6LALqPVUypkzZ3L77bfTt29f+vfvz/PPP09RUREzZswAYNq0acTExDBnzhzAORS+b9++tGvXjrKyMr744gvefvttXn75ZQAsFgv33nsvf/3rX+nQoQNt2rThkUceITo6uuqmzFUNah9KVKAXqXmlfJ2YzrU9os2OJCIizdywYcPo2bMnzz//fJ283vTp08nNzeWTTz6pk9cT11bb+7yNGzeSkpJCz549SUlJ4bHHHsPhcPDAAw9UveaECRP429/+RqtWrejatSvbtm1j7ty5/PznPzflPYqIXLLcJNj3mbN4OLEBOGsBrvCu1WVYeOcaBYTdYbDtxClW7stgVWIG+9MLarysu81SYxri2bspVk1brNpZcaJzZ0VPG5w6XD3t8ug6fMvz6JKzEnJWwn6cBV27q6HdcPAfDNbGu9tlY2CxWGgf7k/7cH9+MbgNpRV2thw/xTcHM1l3IIu9qfnsTnE+Xl5zGB8PG1e0bcHQDqEM6RhG21Df2k+7LMqGHe/ClvmQfbD6eGQP6DsDuk0BL+3m/FNqXYzddNNNZGZmMnv2bNLS0ujZsyfLly+vWqj1xIkTNX4rWFRUxF133UVycjLe3t7Ex8fzzjvvcNNNN1Vd88ADD1BUVMSvfvUrcnNzGTx4MMuXLz9nlyNXY7NamNKnJS+uOsSHm5NVjImIiEijVtv7vNLSUh5++GGOHDmCn58f48eP5+233yYoKKjqmhdffJFHHnmEu+66i4yMDKKjo/n1r3/N7NmzG/rtiYjUXvZh506Se5fCya01z0X3ri7DWrSrcSqvpIJvDmSyal8Gq/dnkFtcUXXOZrXQp3UwI+LDGR4fTvtwv0tbhyq0g/Mx4Fdgr3DubHmmKEveDDmHnY8fXgeLzbme1JndLmP61FzjTOqcl7uNQe1DGdQ+lFnjIKOglG8PZbHuQBbfHMwiq7CMVfsyWLUvA4CYIO+qRfwHtQsl0OcCXx/DcI5S3PKmc/quvdx53MMPuk9xTpeM7tUg77GpsBjGpe4z4Try8/MJDAwkLy+PgICGbUOPZxdx1bNrsFjgu4eGExXo3aCfX0RE6l5paSlHjx6lTZs2Lv9LmrNNnz6d+fPn1zh29OhRCgsLuf/++1m3bh2+vr6MHj2af/zjH4SGhgLw0Ucf8Ze//IVDhw7h4+NDr169WLJkCc8+++w5C6CvXr36RzfPOd9UyrVr13L//fezY8cOQkJCuP322/nrX/+Km5vbj35+X19f1qxZwwMPPMCePXtwd3ena9euvPvuu7Ru3fqi/15+7Otp5j2EXDx9nZqwk9vg8/vALxxaDYTWV0JUgn5gF/MYBmTsdRYOe5c6F8OvYnH+/7TLRIi/FoJiz3qawZGsIlYlZrByXzo/HDuF3VH9o3agtzvDOoUxPD6cqzqGEeRTz6O3SvPg2HpnUXZ4lbMgO5uHP7QZUl2UtWivaXYNyOEw2JdW4FzE/2AmPxw9Rbm9ekqt1QIJsUGnp12G0jM2CLeSs0aHnf31jO7lLMO63QCe/g3/ZlzYxd4/NMiulE1Z6xa+9G8TwqajOSzamsLdV7c3O5KIiNQ1w4CKYnM+t7vPRd+ovvDCCxw4cIBu3brx+OOPO5/u7k7//v355S9/yT/+8Q9KSkp48MEHufHGG1m1ahWpqanccsstPPPMM0yePJmCggLWrVuHYRjcd999JCYmkp+fzxtvvAFASEhIreKnpKQwfvx4pk+fzltvvcW+ffu444478PLy4rHHHvvRz19ZWcl1113HHXfcwXvvvUd5eTmbNm0yb3cnEalbabvgreugNNf55/1fOP/X3Qda9nOWZK0GOj/28DErpTQHhuEsac+MDDu7dLDYoM1QZxnW6Rrwj6g6VV7p4IdjOaxMzGDVvnSOZde8V+gQ7sfwzuGMiI+gd6sg3BpyEXuvQIi/xvkAyD1RPZrsyFooyXH+mzvz7y6g5elpl1dDm2Hg26LhsjZDVquFLtEBdIkO4NdXtaOk3M7Go9l8c3p9soMZhWw7kcv2EzlsXr2Yae6rGWn5ATdOr2Xn4Q89pkLv2yG6p6nvpSlQMVYHpvZpyaajOSzcnMRdw9rphl1EpKmpKIYnTZou/6eT4OF7UZcGBgbi4eGBj49P1S6Cf/3rX+nVqxdPPvlk1XXz5s0jNjaWAwcOUFhYSGVlJddff33VKKzu3btXXevt7U1ZWdmP7j79Y/79738TGxvLv/71LywWC/Hx8Zw8eZIHH3yQ2bNnk5qaesHPn5OTQ15eHtdeey3t2jmnqHTu3PmScoiIi8lIhLcmOUuxlv2cU9FObIDj3zmPHV3rfABY3SCqp7Moa30lxA4An9qV9CLncNghaZOzDEv8FPKSqs/ZPJ3rcnWZCB3H1vj/W1ZhGWv2Z7JqXzrfHMiisKx60X0Pm5UBbUNOT5GMoFULFyp0g1pBn9udD4cD0nZUF2Unvof8ZNj2tvOBBaJ6VI8mi70C3BvPCPrGyNvDxrBO4QzrFA5A+skTpH8zj6jDHxBWcbLquu2OdrxrH84un+H0s7diSG4YA1tU4uepaudy6G+vDozvHsVjS/dwLLuYH46don8bfaMWERHXsGPHDlavXo2fn9855w4fPszo0aMZMWIE3bt3Z8yYMYwePZopU6YQHBxcJ58/MTGRgQMH1vil0aBBgygsLCQ5OZmEhIQLfv6QkBCmT5/OmDFjGDVqFCNHjuTGG28kKiqqTrKJiEmyDsL8iVCc7ZwCdNvHztEtg37n/IE9cx+c+A6Ony7KCk5Cymbn47t/Ol8jvEv11MtWAyEwxtz3JI2DvcI5vTBxKez7HArTq8+5+0KHUc4yrMPoqilphmGQeDKfVfvSWbkvg+1JuZy9GFGonyfD48MYHh/B4A6hjaOgsFqd//aie8GQmVBe7Pw3d3i185GxB1J3OB/fPg9u3tB6YHVRFtFN0y7rg8MBR9fAljeJ2Pc5Ead3OjU8A8huO4mvvcey6GQLtp44RWWOQeKG47y14ThuVgu9Wwc7F/HvEEa3mEDtPlpLjeBfrevz9XTjmh5RfLg5mYWbk1SMiYg0Ne4+zpFbZn3uy1BYWMiECRN4+umnzzkXFRWFzWZjxYoVfPfdd3z11Ve8+OKL/PnPf2bjxo20adPmsj73xfipz//GG2/wu9/9juXLl/PBBx/w8MMPs2LFCq644op6zyYi9SD7MMyfAEUZENkdblvkLMXOsFohoovz0e+XziluucedJdmZsiz7oHMNqIy9sPm/zucFta4uyVpfqfWSpFplmbPsSVzqnDZYcqr6nGcgdBoHnSdA+xHg7lwvuqTczneJziJs9b4MUvNKa7xkt5gAhsdHMCI+nO4xgVgbewnh4QPtRzofAAXpcGSNczTZ4dVQmOZcp+zwKlgB+IZVl2Rth0GANqG7LAVpsO0d2PqW8793Z7TsB32mY+k6mVAPX24GbgYKSiv4/kgO3xzIZN3BTI5lF7PpaA6bjubw3FcHCPZxZ1D7UIZ2CGNIx1Ctg34RVIzVkal9Y/lwczKf70rlsYld8W0MvykQEZGLY7Fc9HRGs3l4eGC326v+3Lt3bz7++GPi4uKqFrv/XxaLhUGDBjFo0CBmz55N69atWbx4MTNnzjzn9Wqrc+fOfPzxxxiGUTVq7Ntvv8Xf35+WLVv+5OcH6NWrF7169WLWrFkMHDiQd999V8WYSGN06rhzpFhBqnPE18+W/PSUSIsFguOcj563OI8VZlZPuzzxnXOtstzjzseO95zX+IbVHFEW2R2stvp8d+JKyovg4ApnGXbgKygvqD7nE+pcd6vLRIgbCm7ORfBP5pawat9xVu3L4NtDWZRVVi+E7n16d8ERncO5ulM4kYFNfFqhfwQk3OR8GIZzFOeZaZfH1kNRJuz60PkACIuvLspaDwLPc0epy/9w2J1/p1vegP3LwDh9r+UZCAk3O6e8RnQ971P9vdwZ1SWCUV1O7xidXcw3B50l2XeHsjlVXMFnO1P5bGcq4FzrbkiHMIZ2DGVAmxZ4e+i/hf9L7U0d6ds6mDahvhzNKuKLXalM7Rv7008SERGpY3FxcWzcuJFjx47h5+fH3XffzWuvvcYtt9zCAw88QEhICIcOHeL999/n9ddfZ/PmzaxcuZLRo0cTHh7Oxo0byczMrFrLKy4uji+//JL9+/fTokULAgMDcXe/+N3i7rrrLp5//nl++9vfcs8997B//34effRRZs6cidVqZePGjRf8/EePHuU///kPEydOJDo6mv3793Pw4EGmTZtWX399IlJf8pJh/rXOdYxCO8K0JZe+uLdfmLPU6DLR+efSfEjeVD31MmWL8wf3xKXOBzgXqo7tX71OWXRvrZnU1JTkwoEvnV/zQ19D5VmjvPyjnaPCOk9wfv2tNuwOgx3JuaxKPMrKfRkkpubXeLmYIG+Gx4czvHM4A9u2wMu9mZYJFguEd3Y+Bt7lHIGXtKl6NNnJbc7iLHMfbHwZrO7Of2tnirLoXiqlz5afWj06LO9E9fHYK5w7S3aZVOvNRlq18OG2Fq257YrWVNodbE/K5ZuDzkX8dyTlcjCjkIMZhcz79igeNiv92gQ7R5N1CKNzlL/WSAcshnH2DOnGyVW28H5p9SGe/XI//eNC+PA3A03LISIil6e0tJSjR4/Spk0bvLwa1w9OBw4c4Pbbb2fHjh2UlJRw9OhRKioqePDBB1m9ejVlZWW0bt2asWPHMnfuXPbt28cf/vAHtm7dSn5+Pq1bt64qsQAyMzO59dZb2bBhA4WFhaxevZphw4Zd8PMfO3aMNm3asG3bNnr27AnA2rVruf/++9mxYwchISHcfvvt/PWvf8XNzY3ExMQLfv709HR+85vfsHHjRrKzs4mKiuL222/n0UcfxWq9+J29fuzr6Sr3EPLj9HVq5PJT4c3xkHMEQtrC9C8goB7XCqwsg5St1VMvkzZCWc3SA5sHxPSpHlUW27/mlE5pHIqynGuFJS517rToqKg+Fxzn3NSh80Tn19pqpaC0gnUHs1iZmMGa/RlkF5VXXW61QO9WwVwdH86IzuF0ilBhcFGKc+DoN9VF2dlTAcH576rNVaenXV4NIfW/TIPLcdjh0ErY8iYcWF49OswrCBJucY4OC6+fzYVyi8v57nA26w5m8s2BLFJyS2qcD/XzdK5N1jGUwe3DCPP3rJccZrnY+wcVY3UoLa+UK59aicOANfcNIy60cUy7ERGRmhpzMSbnUjHW+Onr1IgVZsCb10DWAec6YDO+gMCWDZvBYYf0PdVTL49vcK5xdjaL1bmg+NnrlPmFN2xOuTj5JyHxM2cZdvxbMKqnPBIWf7oMm+CcPmuxcCyriJX7Mli1L51NR3OosFf/+Ovv5cZVHcMY0TmcqzqGE+LrYcIbamJyjlRPuzz6DZTm1TwfHFc9mqzNUPCum81+XFJeSvXosPzk6uOtrjw9Omxi1bp2DcEwDI5kFbHuQCbfHMxiw+FsSipqLpfRJSqAIR2d65P1jQvG061xj/ZTMWaS2+dtYu2BTO65uj33jelkahYREbk0KsaaFhVjjZ++To1UUbZz+mTGXgho6SzFglubncq5ZlLOEWdRdqYsO3Xs3OtatK+5TllwnBb0N8upY7B3KSR+6pw2e7aohOqRYWEdqbA72HzsVNUukkcyi2pc3jbMlxHx4QyPj6BvXDDutosfgSy1ZK90TrU8M5oseROc3mkRcBbS0b2qi7KW/avWfGu07JXOqbxb3oSDX1YXt97BkPB/ztFhYa7RE5RV2tl6PLdqfbLdKTVH1nq5W7mibQuGdAjjqo6htAvza3SjKFWMmeTznanc/e5WogK9WP/gcG2TKiLSCKkYu7Ann3ySJ5988rznhgwZwrJlyxo40U9TMdb46evUCBXnwFsTnQvj+0c5S7GQtmanurD81OrRZCc2OEeY8T8/JvlHVRdlra+EsM7OXTSlfmTuP12GLYW0nTXPxQ6oXjMsOI5TReWsOZDBysQM1h7IpKC0unxxs1oY0DaE4fERDI8Pp41m9ZinrACOfVtdlGXtr3ne3RfiBldPuwzr1HjK6Nwk5+iwbW9Dfkr18daDnaPDOk9w+XUNswrL+PZQFt8ccK5PllFQVuN8dKAXQ07vdDmoXSjBjWCEpYoxk5RV2hnw5EpyiyuY//P+XNUxzNQ8IiJSeyrGLiwnJ4ecnJzznvP29iYmJqaBE/00FWONn75OjUxpHrw1yTlSxDfcWYqFdjA7Ve2UnIITG6vLspPbaq5fBc71gVpdUV2WRfVs/KNdzGQYzgLszMiws0sTi9VZmHSeCPHXYvhHciC9kJX70lmVmMHWE6dwnPVTbYivB1d3cq4VNrhDKAFeF79pjDSgvBQ4sqa6KCvOqnneP6p6NFnbYa43vdle6RwVtmU+HFpx1uiwEOj5f85CrLH9t+80wzDYn17AugNZfHMwk41Hcyg/a6dWiwV6tAxyrk/WIYxerYJccvSlijETPbpkN/M3HOfaHlH86/96mx1HRERqScVY06JirPHT16kRKSuAtydD8g/g0wKmf15vi0o3qPJi526XZ6ZeJv0AFTWn6OHmDS37Vk+9jO0PHhqd9KMcDkjZDHuXOMuwsxdut7o7C5HOE6DTNZR6BPH9kWxW7XOODPvfRcQ7RwU4p0h2DiehZZBm7jQ2Dgek764uyU5sqLmzKDjXAWw7zPn/i1ZX1nr3xjpz6rhzZNi2d6Agtfp4m6HQ+3bn/2fdmtYi9qUVdjYdzeGbA5msO5jF/vSCGuf9PN0Y2K4FQzuEMrRjGK1buMZ/+1SMmWh3Sh7XvrgeDzcrm/40giAf/eZIRKQxUTHWtKgYa/z0dWokyovgnRucP9B6BcH0z5wLoDdF9grn6KYzUy+Pfwcl/zOa1mKD6J411ynzCTElrkuxVzrLxcRPnYvoF5ysPufmDR1GOkeGdRxDerknq/dlsHJfBusPZtVYKNzTzcqg9qEMjw/n6vhwYoIabhFzaQAVpc5/W2eKsv+dTmvzdI7YPDPtMrJH/U5ttlc4d5Tc8qZzh8kzU619QqHXrc5CrEW7+vv8LiY9v7SqJFt/KIucs3Z4BWgV4sOQ0yXZwHYtTBu1qWLMRIZhMP6f60lMzefxSV2ZNjDO7EgiIlILZ4qUuLg4vL11o93YlZSUcOzYMRVjjZi+To1ARQm8e6NzFzrPQLh9iXNR7ebC4XDuvHn2OmV5SedeFxZ/uigbBK0HNvwOnWapLIeja53rhe37HIqzq895+EOnsdB5Ao62I9idVcnKxAxW7ctgV0rNHQ0jA7wY3jmcEfHhXNkuFG+Pxr1jntRCYabz/0Nndrw8ex0vcI5QbXNVdVEWFFs3n/fUMeeuktvegcL06uNthzmnSna6ptlPoXY4DPaczK9axH/L8VM1dn+1WS30ig1iaMcwhnQIpUcDjuhUMWayeeuP8vhne+keE8invx1sdhwREakFu93OgQMHCA8Pp0WLFmbHkcuUnZ1NRkYGHTt2xGar+UOUK95DyLn0dXJxFaXw/i1weJWz5Jj2iXNKYXOXe+J0SXa6LPvfhcYBAls5C7Izo8pCOzaexcZ/SnkxHF7pHBm2fzmUnVVyeQc7C4UuEymKGcz6YwWsSsxg1f4MMs9a8NtigYSWQVVTJLtEBTS6XfGkHhgGZB2sHk12bB2UF9a8pkX70+uTDXeuT+dVi+8d9grY/4VzdNjhVdXHfcOg123Qe5prbyZissKySjYeyWbdwSy+OZDJkaya084Dvd0Z3D6UIR1CGdIxrF5He6oYM1lOUTkDnvyaCrvB8nuHEB/pGrlEROTipKamkpubS3h4OD4+ProRb4QMw6C4uJiMjAyCgoKIioo65xpXvIeQc+nr5MIqy+GD25wLULv7wm0fO4seOVdRFpz4vnqdstSdYNhrXuMT6pwedmbqZWQPsLmZk/dSlObDwa+cI8MOroCK4upzfhEQfy10mUhSQG9WHchh5b4Mvj+cTbm9elFvXw8bQzuGMTw+nGGdwgnzb1prNUk9sFdA8ubqoixlc/VC+OCc1tyyX/Vospg+5/93lXPk9OiwBVCUUX283XDn6LCO45r96LBLkZRTzPpDzpJs/aGsGrvGArQL8+XdO64gIqDuly9RMeYCfvP2FpbvSeMXg9vwyLVdzI4jIiK1YBgGaWlp5Obmmh1FLlNQUBCRkZHnLTdd9R5CatLXyUXZK2DhdNj3mXNtqFsXQpshZqdqPMoKIXlT9dTL5B/OXWzcw8+5iH+rK52FY0wfcHexKf7FObB/mbMMO7wK7GetNRTYCjpPoDL+WrYZHVm5L4tV+9I5kF5zdE+rEB9GdA5nRHwE/doE4+mmKZJyGUpynaPIzky7zDlS87xnAMQNcRZlbYZCxl7n6LAja6qv8Ytwjg7r9TMIadOA4Zu2SruDHcl5rDvoXJ9s24lTBPl4sPnPI7HWw/RKFWMuYNW+dH7+5mZCfD34ftYIPNxcb/tSERH5cXa7nYqKCrNjyCVyd3c/Z/rk2Vz1HkJq0tfJBdkrYdEvYc9i5yLY//e+c1SFXLrKMji5/ax1yr6vOf0QwObhXLvtzDplsf3BO6jhsxakOwvRxKVwdF3NkW8t2kPniRS0Hc/q/GhW7ctgzYFMcourv5farBb6tg5mROdwhsdH0C7MVyOzpf6cOl49muzoWig5dYELLdB+xOnRYWPBZs6C8c1JXkkFR7OK6BkbVC+vr2LMBVTaHVz51CoyCsp45bY+jO0WaXYkEREROYur3kNITfo6uRiHHT65E3Z+AFZ3uPld6Dja7FRNj8PuHMly9jplhWn/c5EFIrrVXKfMv55+5sg94dxFMnGps7TjrB8jI7pjdL6WpMhRLE8PZOW+TDYfP4XdUX1NkI87wzqGMbxzBFd1CCPQR6WDmMBhh9Qd1UVZ0kbnwv1nRocFtzY7odQhFWMuYs6yRF5de4SRncN5/fZ+ZscRERGRs7jyPYRU09fJhTgc8OlvnTu0Wd3gxrcg/hqzUzUPhgGnjp5VlH137hQxcC4KfmbqZauBzj9f6mis7MOwd4mzDDu5rea5mD5UdprAdr8hfJ7izap9GRzPLq5xSccIP4bHRzCiczi9YoNws2kGjbgYe4Xzv2UasdgkXez9QyNaybFxmtonllfXHmH1/kwyCkoJ96/7BeVEREREROqdYcAXf3SWYhYr3PC6SrGGZLE4S66QttDrVuexgjTn+mRnyrK03c6yLOcIbH/HeY1f5OmS7HRZFt4FrBeYYm4YzlFqe5c6y7CMvWcHgNZXUth2PGus/fn8uI1vvs6kqDy96goPm5Ur2rVw7iIZH05siE/9/F2I1BVNlxRUjNW79uF+9G4VxNYTuSzemsKvr2pndiQRERERkdoxDFj+EGyeB1hg8n+g62SzU4l/pPPrcOZrUZILSZuqp16e3OqcfrlnsfMB4BkIrQZUr1MW3RPSd1eXYWePQrO6YbQZysmoUXxR0ZvPj9jZsTwXw8isuiTM35PhncIZ3jmcwe1D8fXUj5gi0rjov1oNYGrfWLaeyGXhlmR+NbStFpYUERERkcbDMGDFI7DxFeefJ70EPaaam0nOzzvIud7bmTXfKkogZWv11MukTc4F/Q9+5XyAc/Sf4ah+DZsn9rbD2Rc8jI8Ku7PsUClpe0qB7KpLuscEMjw+nJGdI+gaHVAvu8mJiDQUFWMN4NoeUfzl0z0cyihke1IuvVoFmx1JREREROTirPorfPei8+Nrn6+exieuz90b4gY5H+DcTTR9V80F/YuzwN2X4rgRbPYZwrs5nVidWExZpQPIBcDb3caQDqGM6BzO1Z3CCQ/Q8jAi0nSoGGsA/l7ujOsWxeJtKSzckqxiTEREREQah7XPwLrnnB+Pexb6zjA3j1wemxtE94LoXhhX3Mnu5Dw2bNvOZ4cr2Lmr/PRFhQDEBHkzsnM4wztHMKBNCF7uF1iXTESkkVMx1kCm9m3J4m0pfLr9JI9c0wVvD31jEREREREXtv4fsPpvzo9H/w0G/MrcPHLZHA6DbUm5LN+dyrLdaSSfKqk6Z7VAn9bBVbtIdgj30xIwItIsqBhrIFe0aUHLYG+ST5Xw5Z40rusVY3YkEREREZHz2/ASfP2Y8+MRs+HKe0yNI5fO7jDYdDSH5btTWb4njfT8sqpz3u42hnUKY3TXCIZ1DCfY18PEpCIi5lAx1kCsVgtT+rTk+a8PsnBLkooxEREREXFNm16DL//k/HjYLBjyR3PzSK1V2B18dzib5btT+WpPOtlF5VXn/DzdGNE5nHHdIrmqY7hmsohIs6dirAFN6dOSF1Ye5LvD2STlFBMb4mN2JBERERGRalvehC/uc348eCZc9aCpceTilVbYWXcwi2W7U/l6bzr5pZVV54J83BnVOYJx3SMZ1D4UTzeVYSIiZ6gYa0Atg324sl0Lvj2Uzcdbk7l3ZEezI4mIiIiIOG1/Fz691/nxwHucUyi1xpRLKy6vZPW+TJbtTmX1vgyKyu1V50L9PBnTNYJx3aIY0DYEd5vVxKQiIq5LxVgDm9onlm8PZfPRlmR+N7wDVqtuNkRERETEZLs+giV3Awb0/xWM/qtKMReVX1rBysR0lu1KY+2BTMoqHVXnogK9GNstknHdoujTOhibftYQEflJKsYa2Jiukfh7upF8qoTvj2ZzZbtQsyOJiIiISHO2dwks+hUYDugzHcY9o1LMxeQUlbNibxrLdqfx7aEsKuxG1blWIT6M6xbJuO5RJLQM1E6SIiK1pGKsgXl72JjQM5p3N55g4eZkFWMiIiIiYp59X8BHPwfDDj1vhWv+oVLMRWQUlPLlnnSW7Upl49Ec7I7qMqx9uB/jukUytlskXaICVIaJiFwGFWMmmNqnJe9uPMGy3an8ZVJXArzczY4kIiIiIs3NwRXw4TRwVEL3G2Hii2DVOlRmSsktYfnuNJbvTmXz8VMY1V0YXaICTo8Mi6R9uL95IUVEmhgVYyboGRtE+3A/DmUU8vnOVG7p38rsSCIiIiLSnBxeDe/fCo4K6HIdXPcyWLVToRmOZRWx7HQZtiM5r8a5nrFBVSPDWrfwNSmhiEjTpmLMBBaLhal9WjJn2T4Wbk5SMSYiIiIiDefYenjvFrCXQadr4IbXwaYfCxqKYRgczChk2a40lu1OZV9aQdU5iwX6xYUwrlskY7pGEh3kbWJSEZHmQd8BTTK5dwzPfLmfrSdyOZRRoOHQIiIiIlL/TnwPC26EyhLoMBqmvgE2LetR3wzDYM/JfJbtTmXZ7jSOZBZVnbNZLQxs24Jx3SMZ3SWSMH9PE5OKiDQ/KsZMEu7vxdWdwvg6MYOFW5KZNa6z2ZFEREREpClL3gLvTIGKImh7Ndz4NriphKkvDofB9uRclu1KZfmeNJJySqrOedisDO4QythukYzqHEGwr4eJSUVEmjcVYyaa0ieWrxMzWLQ1hftHd8LNpsVORURERKQenNwOb0+G8gKIGwI3vwvuXmananLsDoMfjuWcXkA/jbT80qpzXu5WhnUMZ1z3SIbHh+OvDbhERFyCijETDY8PJ8TXg8yCMr45mMnw+AizI4mIiIhIU5O2G96+DsryIPYKuOV98PAxO1WTUWF3sOFwNst2p7FibxpZheVV5/w83RgeH864bpFc1SkMHw/9+CUi4mr0X2YTebhZmdwrhv+uP8qHPySrGBMRERGRupWxD96aBCWnIKYv3LoQPP3MTtXolVbYWX8wi2W70/g6MZ28koqqc4He7ozqEsG4bpEMah+Kl7t2+xQRcWUqxkw2tW9L/rv+KCv3pZNTVE6I1hcQERERkbqQdQjemgjFWRDVE277GLwCzE7VaBWXV7JmfybLdqexKjGdonJ71blQPw9Gd41kXLdIrmjbAnctkSIi0mioGDNZfGQA3WMC2ZWSxyfbUvj54DZmRxIRERGRxi7nCMyfAIXpENENfrYYvIPMTtXo5JdWsCoxg2W7U1l7IJPSCkfVuahAL8Z0jWRst0j6xYVgs1pMTCoiIpdKxZgLmNq3JbtS8vhwcxIzBsVhseibqoiIiIhcotwTMH8iFJyEsHiYtgR8QsxO1WicKipnxd50lu1O5dtD2ZTbq8uw2BBvxnWLYly3SBJaBmFVGSYi0uipGHMBExOi+evniexLK2DPyXy6xQSaHUlEREREGqO8FHjzWshLghbtYdpS8A01O5XLyygo5as9zjLs+yM52B1G1bl2Yb6M6xbF2G6RdI0O0C+xRUSaGBVjLiDIx4PRXSL4bGcqCzcnqRgTERERkdorSHNOn8w9DsFt4PZPwV+bO13IydwSlu9OY/nuNH44noNR3YXROSqAcd2ca4Z1iPA3L6SIiNQ7FWMuYmrfWD7bmcqSHSf50zWd8XTT7jUiIiIicpEKM53TJ3MOQ1ArZykWEG12KpdzPLuIZbvTWLY7jR1JuTXOJcQGMa5bJGO7RhIX6mtOQBERaXAqxlzE4PahRAV6kZpXytd7M7imR5TZkURERESkMSjKhrcmQdZ+CIhxlmJBsWanchkH0wuqyrDE1Pyq4xYL9GsdwthuzgX0o4O8TUwpIiJmUTHmImxWCzf0bsm/Vh/iw81JKsZERERE5KeVnIK3J0HGHvCLdJZiwXFmpzKVYRjsOZnP8t1pLNudyuHMoqpzNquFK9qGMK5bFKO7RhDu72ViUhERcQUqxlzIlD7OYmzdwUzS8kqJDNQ3ahERERG5gNI8ePt6SNsFvmFw+1Jo0c7sVKZwOAy2J+dWrRl2Iqe46py7zcLg9qGM6xbFqC4RBPt6mJhURERcjYoxFxIX6kv/uBA2Hcvh463J3H11e7MjiYiIiIgrKiuABVPh5FbwDnHuPhnWyexUDcruMPjhWE5VGZaWX1p1zsvdylUdwxjXLYrhncMJ8HI3MamIiLgyFWMuZmrflmw6lsPCzUncNaydtoMWERERkZrKi+DdmyBpI3gFwbQlENHF7FQNosLu4Psj2SzbncZXe9LIKiyvOufrYWN45wjGdYtkWKcwfDz0o46IiPw0fbdwMeO7R/Ho0j0cyy5m8/FT9IsLMTuSiIiIiLiKihJ47xY4/i14BsDPFkNUD7NT1auySjvrD2axbHcaK/amk1dSUXUuwMuNUV0iGdctksEdQvFy187uIiJSOyrGXIyvpxvXdI9i4ZZkFm5OUjEmIiIiIk6VZfDBbXB0LXj4wW0fQ0xvs1PVi+LyStbuz2TZ7jRW7cugsKyy6lwLXw9Gd3WWYQPbtcDdZjUxqYiINHYqxlzQ1L6xLNySzOc7U3l0Qld8PfVlEhEREWnWKsvhw9vh0Nfg7gP/9yHE9jc7VZ0qKK1g1b4Mlu1KY82BDEorHFXnIgI8Gds1krHdoujfJgSbVcuNiIhI3VDj4oL6xQUT18KHY9nFfLErlal9Y82OJCIiIiJmsVfAxz+HA8vAzQtueR/iBpmdqs58uuMki7elsP5gFuX26jKsZbA347o5y7BesUFYVYaJiEg9UDHmgiwWC1P7xvLsl/tZuCVZxZiIiIhIc+Www+JfQ+KnYPOAmxdA26vMTlVnPtmWwr0fbK/6c9swX8Z1i2Rctyi6RgdoIyoREal3KsZc1PW9Y/j7V/vZdDSH49lFtG7ha3YkEREREWlIDgcsuRt2fwxWd7jxbWg/0uxUdeqDH5IAmJgQzT3D29Mh3E9lmIiINKhLWqnypZdeIi4uDi8vLwYMGMCmTZsueO1rr73GkCFDCA4OJjg4mJEjR55z/fTp07FYLDUeY8eOvZRoTUZUoDeDO4QB8NGWZJPTiIiIiEiDcjjgs9/DjvfAYoOpb0CnpnV/nJZXyvdHswG4f0wnOkb4qxQTEZEGV+ti7IMPPmDmzJk8+uijbN26lYSEBMaMGUNGRsZ5r1+zZg233HILq1evZsOGDcTGxjJ69GhSUlJqXDd27FhSU1OrHu+9996lvaMm5Ma+LQFnMWZ3GCanEREREZEGYRjwxX2w9S2wWOGG16DzBLNT1blPd5zEMKBv62BiQ3zMjiMiIs1UrYuxuXPncscddzBjxgy6dOnCK6+8go+PD/PmzTvv9QsWLOCuu+6iZ8+exMfH8/rrr+NwOFi5cmWN6zw9PYmMjKx6BAcHX9o7akJGdo4g0Nud1LxSvj2UZXYcEREREalvhgHLZ8Hm/wIWuO4V6HaD2anqxZIdzl+UT+oZbXISERFpzmpVjJWXl7NlyxZGjqxe28BqtTJy5Eg2bNhwUa9RXFxMRUUFISEhNY6vWbOG8PBwOnXqxJ133kl2dvYFX6OsrIz8/Pwaj6bIy91WdaOwUNMpRURERJo2w4CvH4WNLzv/PPFFSLjJ3Ez15FBGIbtT8rFZLYzvHmV2HBERacZqVYxlZWVht9uJiIiocTwiIoK0tLSLeo0HH3yQ6OjoGuXa2LFjeeutt1i5ciVPP/00a9euZdy4cdjt9vO+xpw5cwgMDKx6xMY23V0bp/Zxvrcv96SRV1xhchoRERERqTern4RvX3B+fM1c6P0zc/PUo6U7TgIwpEMoLfw8TU4jIiLN2SUtvn+pnnrqKd5//30WL16Ml5dX1fGbb76ZiRMn0r17d6677jo+++wzfvjhB9asWXPe15k1axZ5eXlVj6SkpAZ6Bw2vW0wA8ZH+lFc6WLoj5aefICIiIiKNz9pn4ZtnnB+PfRr6/cLcPPXIMAyWbtc0ShERcQ21KsZCQ0Ox2Wykp6fXOJ6enk5kZOSPPve5557jqaee4quvvqJHjx4/em3btm0JDQ3l0KFD5z3v6elJQEBAjUdTZbFYmNrXOWpM0ylFREREmqBvX4DVf3V+POpxuOI35uapZzuT8ziWXYyXu5XRXX78ZwgREZH6VqtizMPDgz59+tRYOP/MQvoDBw684POeeeYZnnjiCZYvX07fvn1/8vMkJyeTnZ1NVJTWGwC4rmc0blYLO5Pz2J9WYHYcEREREakr378MK2Y7Px7+MAz6vbl5GsCS7c5plKO6ROLr6WZyGhERae5qPZVy5syZvPbaa8yfP5/ExETuvPNOioqKmDFjBgDTpk1j1qxZVdc//fTTPPLII8ybN4+4uDjS0tJIS0ujsLAQgMLCQu6//36+//57jh07xsqVK5k0aRLt27dnzJgxdfQ2G7cWfp6M6BwOwMLNTXfaqIiIiEiz8sPrsPwh58dDH4Ch95ubpwHYHQaf7nQWY5MSNI1SRETMV+ti7KabbuK5555j9uzZ9OzZk+3bt7N8+fKqBflPnDhBampq1fUvv/wy5eXlTJkyhaioqKrHc889B4DNZmPnzp1MnDiRjh078otf/II+ffqwbt06PD21EOcZN56eTrl4WwoVdofJaURERETksmx9Cz7/o/PjQffC1X8yNU5D+f5INpkFZQR6uzO0Y5jZcURERLikscv33HMP99xzz3nP/e+C+ceOHfvR1/L29ubLL7+8lBjNylUdwwjz9ySzoIxV+zIY01XrMYiIiIg0Sjveh6W/c358xV0w8jGwWEyN1FCWnF50f3z3KDzcGnQfMBERkfPSd6NGws1m5fpeMQAs3KxF+EVEREQapd0fwyd3Agb0+yWMebLZlGKlFXaW7U4DtBuliIi4DhVjjcjUvi0BWL0/g4yCUpPTiIiIiEit7F0KH98BhgN6T4NxzzabUgxgzf5MCkoriQzwon9ciNlxREREABVjjUr7cH96tQrC7jD4ZFuK2XFERERE5GLtXwYf/RwMOyTcAte+ANbmdSu+dIfz/nViz2is1uZTCIqIiGtrXt+Nm4CpfZyL8C/cnIxhGCanEREREZGfdPBr+HAaOCqg2w0w6aVmV4oVlFbwdWIGABO1G6WIiLiQ5vUduQm4NiEKL3crBzMK2ZGcZ3YcEREREfkxR9bAB7eCvRw6T4TJr4LVZnaqBvflnnTKKx20C/Ola3SA2XFERESqqBhrZAK83BnXLQqADzcnmZxGREREmpqXXnqJuLg4vLy8GDBgAJs2bbrgtRUVFTz++OO0a9cOLy8vEhISWL58+TnXpaSkcNttt9GiRQu8vb3p3r07mzdvrs+34RqOfQvv3gyVpdBxHNzwX7C5m53KFGd2o5zUMwZLM1pXTUREXJ+KsUZoah/nIvyf7jhJaYXd5DQiIiLSVHzwwQfMnDmTRx99lK1bt5KQkMCYMWPIyMg47/UPP/wwr776Ki+++CJ79+7lN7/5DZMnT2bbtm1V15w6dYpBgwbh7u7OsmXL2Lt3L3//+98JDg5uqLdljqRN8O6NUFkC7UfCjfPBzcPsVKbILCjj20NZgKZRioiI61Ex1ghd0bYFLYO9KSit5Ms9aWbHERERkSZi7ty53HHHHcyYMYMuXbrwyiuv4OPjw7x58857/dtvv82f/vQnxo8fT9u2bbnzzjsZP348f//736uuefrpp4mNjeWNN96gf//+tGnThtGjR9OuXbuGelsNL2ULvHMDlBdCm6vgpnfAzdPsVKb5fOdJHAYkxAYRF+prdhwREZEaVIw1QlarhRt6O0eNLdycbHIaERERaQrKy8vZsmULI0eOrDpmtVoZOXIkGzZsOO9zysrK8PLyqnHM29ub9evXV/156dKl9O3bl6lTpxIeHk6vXr147bXX6udNuILUHfD2ZCjLh9aD4Jb3wd3b7FSmWrLjJACTNFpMRERckIqxRmrK6emU3x7OIvlUsclpREREpLHLysrCbrcTERFR43hERARpaecfoT5mzBjmzp3LwYMHcTgcrFixgkWLFpGamlp1zZEjR3j55Zfp0KEDX375JXfeeSe/+93vmD9//gWzlJWVkZ+fX+PRKKTvgbeug9I8iB0A//cBePiYncpUx7OL2HYiF6sFru0RZXYcERGRc6gYa6RiQ3y4sl0LDAM+3pJidhwRERFphl544QU6dOhAfHw8Hh4e3HPPPcyYMQOrtfoW0+Fw0Lt3b5588kl69erFr371K+644w5eeeWVC77unDlzCAwMrHrExsY2xNu5PJn7Yf5EKMmB6N5w60Lw9Dc7lemWbneOFruyXSjhAV4/cbWIiEjDUzHWiE3t6xw19tHWJBwOw+Q0IiIi0piFhoZis9lIT0+vcTw9PZ3IyMjzPicsLIxPPvmEoqIijh8/zr59+/Dz86Nt27ZV10RFRdGlS5caz+vcuTMnTpy4YJZZs2aRl5dX9UhKcvGduLMPO0ux4iyI7AE/WwRegWanMp1hGHxyejfKiT01jVJERFyTirFGbGzXKPw93UjKKWHj0Ryz44iIiEgj5uHhQZ8+fVi5cmXVMYfDwcqVKxk4cOCPPtfLy4uYmBgqKyv5+OOPmTRpUtW5QYMGsX///hrXHzhwgNatW1/w9Tw9PQkICKjxcFk5R2H+BChMg/Cu8LNPwLuJ77h5kfam5nM4swgPNytju52/XBURETGbirFGzNvDxrWnFzFduNnFf5MqIiIiLm/mzJm89tprzJ8/n8TERO68806KioqYMWMGANOmTWPWrFlV12/cuJFFixZx5MgR1q1bx9ixY3E4HDzwwANV1/zhD3/g+++/58knn+TQoUO8++67/Oc//+Huu+9u8PdX53JPOEeK5adAaCeYtgR8W5idymWcmUY5Ij6cAC93k9OIiIicn4qxRu7MdMovdqdSUFphchoRERFpzG666Saee+45Zs+eTc+ePdm+fTvLly+vWpD/xIkTNRbWLy0t5eGHH6ZLly5MnjyZmJgY1q9fT1BQUNU1/fr1Y/Hixbz33nt069aNJ554gueff55bb721od9e3co/6RwplncCQtrB7UvBL8zsVC7D4TBYemY3Sk2jFBERF2YxDKPRL06Vn59PYGAgeXl5rj3Uvh4YhsHIuWs5nFnEU9d35+b+rcyOJCIi0mg053uIxsTlvk4FafDmNZB9CILjYPoXEBhjdiqXsvFINjf953v8Pd344eGReLnbzI4kIiLNzMXeP2jEWCNnsViY2te5U9PCLckmpxERERFp4oqy4K1JzlIsMBZu/1Sl2HksOT1abGy3SJViIiLi0lSMNQHX94rBZrWw5fgpDmUUmh1HREREpGkqznGWYpn7wD/aOX0ySKP1/1d5pYMvdjmn3E7qqdJQRERcm4qxJiA8wIthHZ1rWnykUWMiIiIida8kF96+DtJ3g1+Ec6RYSFuzU7mkdQczyS2uINTPk4HttBmBiIi4NhVjTcSZRfgXbU2m0u4wOY2IiIhIE1KaD+9cD6k7wCcUpi2F0PZmp3JZS07vRjkhIQqb1WJyGhERkR+nYqyJGB4fQYivBxkFZXxzMNPsOCIiIiJNQ1khLJgKKVvAOximLYHweLNTuayiskpW7E0HNI1SREQaBxVjTYSHm5XrTt98LNys6ZQiIiIil628GN67GZK+B69A+NknENnN7FQu7evEdEoq7LRu4UNCy0Cz44iIiPwkFWNNyJnplF8nppNTVG5yGhEREZFGrKIU3r8Fjq0DD3+4bTFE9zQ7lcs7M41yUkI0FoumUYqIiOtTMdaEdI4KoFtMABV2gyXbU8yOIyIiItI4VZbBB7fBkTXg7gu3fQQt+5idyuXlFJXzzQHnkh4Te0abnEZEROTiqBhrYm7sGwvAh5pOKSIiIlJ79gpYOAMOrQA3b7j1Q2h1hdmpGoUvdqVS6TDoGh1A+3B/s+OIiIhcFBVjTczEhGg8bFYSU/PZnZJndhwRERGRxsNeCR//AvZ/DjZPuOU9iBtsdqpGY+mZaZQaLSYiIo2IirEmJsjHg1FdIwD4aItGjYmIiIhcFIcdFv8a9i4BmwfcvADaXW12qkYjJbeETcdysFhgQoKKMRERaTxUjDVBU/s4F+H/ZHsKZZV2k9OIiIiIuDiHA5bcA7s/AqsbTJ0PHUaZnapROTNarH9cCFGB3ianERERuXgqxpqgIR3CiAzwIre4gq/3ZpgdR0RERMR1ORzw2b2w412w2GDKPIgfb3aqRufMxk+TesaYnERERKR2VIw1QTarhRv6OG9KFm5JMjmNiIiIiAtb8QhsnQ8WK1z/H+gyyexEjc7+tAL2pRXgbrMwvnuk2XFERERqRcVYEzWlj3N3ym8OZJKWV2pyGhEREREX1WkcePjDpJeg+xSz0zRKS3c4R4td1TGcIB8Pk9OIiIjUjoqxJqpNqC/94oJxGLBomxbhFxERETmvuMHw++3Q8//MTtIoGYbBEu1GKSIijZiKsSZsal/nqLGFm5MxDMPkNCIiIiIuyjfU7ASN1tYTuSSfKsHHw8bIzhFmxxEREak1FWNN2DXdo/DxsHE0q4gtx0+ZHUdEREREmpilpxfdH9M1Em8Pm8lpREREak/FWBPm6+nG+O5RgHPUmIiIiIhIXam0O/hsZyoAEzWNUkREGikVY03c1D4tAfhs50mKyytNTiMiIiIiTcW3h7PJLionxNeDwe01HVVERBonFWNNXP82IcS18KGo3M4Xu9LMjiMiIiIiTcSS09Mor+kehbtNP1aIiEjjpO9gTZzFYmHK6VFjCzcnmZxGRERERJqC0go7X+52/tJVu1GKiEhjpmKsGbi+d0ssFth4NIfj2UVmxxERERGRRm5lYgZF5XZigrzp3SrY7DgiIiKXTMVYMxAd5F217sNHW7QIv4iIiIhcnjPTKCf2jMZqtZicRkRE5NKpGGsmbuwbC8DHW5KxOwyT04iIiIhIY5VXXMGa/ZmAplGKiEjjp2KsmRjVJYIALzdO5pXy3eEss+OIiIiISCO1fE8q5XYHnSL8iY8MMDuOiIjIZVEx1kx4uduY1DMGgIWbNZ1SRERERC7Nku0nAec0ShERkcZOxVgzcmY65fI9aeQVV5icRkREREQam/T8UjYcyQZgYoKKMRERafxUjDUj3WICiI/0p7zSwdKdJ82OIyIiIiKNzKc7TmIY0Kd1MLEhPmbHERERuWwqxpoRi8XClD4tAfhoc5LJaURERESksTkzjVKL7ouISFOhYqyZmdwrBjerhR3JeexPKzA7joiIiIg0EoczC9mVkofNamF89yiz44iIiNQJFWPNTAs/T0Z0DgdgoUaNiYiIiMhFWnp6tNiQDqGE+nmanEZERKRuqBhrhqb2cS7C/8n2FCrsDpPTiIiIiIirMwyDpTs0jVJERJoeFWPN0LBOYYT6eZJVWM7qfRlmxxERERERF7crJY+jWUV4uVsZ1SXS7DgiIiJ1RsVYM+Rms3J97xgAFm5JNjmNiIiIiLi6M4vuj+wcgZ+nm8lpRERE6o6KsWZq6undKVftyyCzoMzkNCIiIiLiquwOg0+rplHGmJxGRESkbqkYa6Y6RPjTMzYIu8Pgk20pZscRERERERe18Ug2GQVlBHq7c1XHMLPjiIiI1CkVY83Y1L7OUWMLtyRhGIbJaURERETEFZ2ZRjm+eyQebvrxQUREmhZ9Z2vGJiRE4+lm5UB6ITuT88yOIyIiIiIupqzSzhe7UwGYmKBplCIi0vSoGGvGArzcGdfNuavQh5uTTE4jIiIiIq5mzf5MCkoriQzwon+bELPjiIiI1DkVY83c1L6xACzdcZLSCrvJaURERETElSw9PY1yQkIUNqvF5DQiIiJ175KKsZdeeom4uDi8vLwYMGAAmzZtuuC1r732GkOGDCE4OJjg4GBGjhx5zvWGYTB79myioqLw9vZm5MiRHDx48FKiSS0NbNuCmCBvCkor+XJPmtlxRERERMRFFJRW8HViOqDdKEVEpOmqdTH2wQcfMHPmTB599FG2bt1KQkICY8aMISMj47zXr1mzhltuuYXVq1ezYcMGYmNjGT16NCkp1TshPvPMM/zzn//klVdeYePGjfj6+jJmzBhKS0sv/Z3JRbFaLdzQ5/Qi/JuTTU4jIiIiIq7iqz3plFU6aBvmS9foALPjiIiI1ItaF2Nz587ljjvuYMaMGXTp0oVXXnkFHx8f5s2bd97rFyxYwF133UXPnj2Jj4/n9ddfx+FwsHLlSsA5Wuz555/n4YcfZtKkSfTo0YO33nqLkydP8sknn1zWm5OLM/V0Mfbt4SxScktMTiMiIiIirmDJDuc0ykkJMVgsmkYpIiJNU62KsfLycrZs2cLIkSOrX8BqZeTIkWzYsOGiXqO4uJiKigpCQpyLdx49epS0tLQarxkYGMiAAQMu+JplZWXk5+fXeMiliw3xYWDbFhgGfLxFo8ZEREREmrvMgjK+PZQFwMSe0SanERERqT+1KsaysrKw2+1ERETUOB4REUFa2sWtT/Xggw8SHR1dVYSdeV5tXnPOnDkEBgZWPWJjY2vzNuQ8pvZ1jhr7aEsyDodhchoRERERMdMXu1KxOwwSWgbSJtTX7DgiIiL1pkF3pXzqqad4//33Wbx4MV5eXpf8OrNmzSIvL6/qkZSUVIcpm6dx3aLw83TjRE4xG4/mmB1HREREREy0ZLtzPeCJWnRfRESauFoVY6GhodhsNtLT02scT09PJzIy8kef+9xzz/HUU0/x1Vdf0aNHj6rjZ55Xm9f09PQkICCgxkMuj7eHjQkJUQAs3KKiUURERKS5OpFdzNYTuVgtMKFHlNlxRERE6lWtijEPDw/69OlTtXA+ULWQ/sCBAy/4vGeeeYYnnniC5cuX07dv3xrn2rRpQ2RkZI3XzM/PZ+PGjT/6mlL3pvRxTkldtiuNwrJKk9OIiIiIiBmW7nCOFruyXSjhAZc+y0NERKQxqPVUypkzZ/Laa68xf/58EhMTufPOOykqKmLGjBkATJs2jVmzZlVd//TTT/PII48wb9484uLiSEtLIy0tjcLCQgAsFgv33nsvf/3rX1m6dCm7du1i2rRpREdHc91119XNu5SL0rtVEG3DfCmpsPP5zpNmxxERERGRBmYYBp9sd94HatF9ERFpDtxq+4SbbrqJzMxMZs+eTVpaGj179mT58uVVi+efOHECq7W6b3v55ZcpLy9nypQpNV7n0Ucf5bHHHgPggQceoKioiF/96lfk5uYyePBgli9fflnrkEntWSwWbuwby1PL9vHh5mRu6tfK7EgiIiIi0oASUws4lFGIh5uVsd1+fKkUERGRpsBiGEaj34IwPz+fwMBA8vLytN7YZcrIL2XgU6uwOwxW/vEq2oX5mR1JRESk3ugeonHQ16nhzFmWyKtrjzC2aySv/KyP2XFEREQu2cXePzTorpTi+sIDvLiqYxgAH21JNjmNiIiIiDQUh8Pg09PTKCdpGqWIiDQTKsbkHFP7tARg0dZkKu0Ok9OIiIiISEPYfPwUJ/NK8fd04+r4cLPjiIiINAgVY3KOEZ0jCPH1ID2/jHUHs8yOIyIiIiINYMl2526UY7pF4uVuMzmNiIhIw1AxJufwcLNWDZ9fuCXJ5DQiIiIiUt/KKx18visV0DRKERFpXlSMyXlN7RMLwNd7MzhVVG5yGhERERGpT+sPZZJbXEGonycD27YwO46IiEiDUTEm59UlOoCu0QGU2x1Vw+pFREREpGlacnrR/Wt7ROFm048IIiLSfOi7nlzQjX2do8Y+3KzdKUVERESaquLySr7akw5oGqWIiDQ/Ksbkgib1jMbDZmVvaj57TuaZHUdERERE6sGKvemUVNhpFeJDz9ggs+OIiIg0KBVjckFBPh6M6hIBwEKNGhMRERFpkpaenkY5qWc0FovF5DQiIiINS8WY/KgpfVsC8Mn2FMoq7SanEREREZG6dKqonLUHMgFNoxQRkeZJxZj8qKEdwogM8CK3uIKViRlmxxERERGROvTF7lQqHQZdogJoH+5vdhwREZEGp2JMfpTNauH63jEALNycZHIaEREREalLS86aRikiItIcqRiTnzSlj3M65doDmaTnl5qcRkRERETqwsncEjYdzcFigQkJKsZERKR5UjEmP6ltmB99WwfjMODjrVqEX0RERKQp+HSHc7RY/7gQooO8TU4jIiJiDhVjclFu7BsLwEebkzEMw+Q0IiIiInK5PqmaRhljchIRERHzqBiTizK+RxTe7jaOZBWx9cQps+OIiIiIyGU4kF5AYmo+7jYL47pFmh1HRETENCrG5KL4eboxvnsUAAs3azqliIiISGO29PRosas6hhHs62FyGhEREfOoGJOLdmNf5yL8n+44SXF5pclpRERERORSGIbBkh0pAEzUNEoREWnmVIzJRevfJoTWLXwoKrezbFea2XFERERE5BJsS8olKacEHw8bIzuHmx1HRETEVCrG5KJZLBam9HaOGlu4JcnkNCIiIiJyKc5MoxzdJQIfDzeT04iIiJhLxZjUyg19WmKxwPdHcjiRXWx2HBERERGphUq7g892ajdKERGRM1SMSa1EB3kzuH0oAB9p1JiIiIhIo/Ld4WyyCssJ9nFncIdQs+OIiIiYTsWY1NrUvrEAfLw1BYfDMDmNiIiI1KWXXnqJuLg4vLy8GDBgAJs2bbrgtRUVFTz++OO0a9cOLy8vEhISWL58+QWvf+qpp7BYLNx77731kFwuxpLT0yiv6RGFu00/CoiIiOi7odTa6C4RBHi5kZJbwneHs82OIyIiInXkgw8+YObMmTz66KNs3bqVhIQExowZQ0ZGxnmvf/jhh3n11Vd58cUX2bt3L7/5zW+YPHky27ZtO+faH374gVdffZUePXrU99uQCyitsPPlHucGSppGKSIi4qRiTGrNy93GxJ7RgBbhFxERaUrmzp3LHXfcwYwZM+jSpQuvvPIKPj4+zJs377zXv/322/zpT39i/PjxtG3bljvvvJPx48fz97//vcZ1hYWF3Hrrrbz22msEBwc3xFuR81i1L4PCskpigrzp00pfBxEREVAxJpfoxtPTKZfvTiOvpMLkNCIiInK5ysvL2bJlCyNHjqw6ZrVaGTlyJBs2bDjvc8rKyvDy8qpxzNvbm/Xr19c4dvfdd3PNNdfUeO0fU1ZWRn5+fo2HXL4l21MAmJAQjdVqMTmNiIiIa1AxJpeke0wgnSL8Kat08OmOk2bHERERkcuUlZWF3W4nIiKixvGIiAjS0tLO+5wxY8Ywd+5cDh48iMPhYMWKFSxatIjU1NSqa95//322bt3KnDlzLjrLnDlzCAwMrHrExsZe2puSKnklFazelwnApNMj/0VERETFmFwii8XC1L4tAVi4JdnkNCIiImKGF154gQ4dOhAfH4+Hhwf33HMPM2bMwGp13mImJSXx+9//ngULFpwzsuzHzJo1i7y8vKpHUpKWbrhcX+5Oo9zuoGOEH/GR/mbHERERcRkqxuSSXdcrBjerhR1JuRxILzA7joiIiFyG0NBQbDYb6enpNY6np6cTGRl53ueEhYXxySefUFRUxPHjx9m3bx9+fn60bdsWgC1btpCRkUHv3r1xc3PDzc2NtWvX8s9//hM3Nzfsdvt5X9fT05OAgIAaD7k8S3Y4p1FO6hmDxaJplCIiImeoGJNLFurnyfD4cAAWbtZvckVERBozDw8P+vTpw8qVK6uOORwOVq5cycCBA3/0uV5eXsTExFBZWcnHH3/MpEmTABgxYgS7du1i+/btVY++ffty6623sn37dmw2W72+J3HKyC+t2kl8YoKmUYqIiJzNzewA0rhN7RvLV3vTWbwthQfGxuNuU9cqIiLSWM2cOZPbb7+dvn370r9/f55//nmKioqYMWMGANOmTSMmJqZqvbCNGzeSkpJCz549SUlJ4bHHHsPhcPDAAw8A4O/vT7du3Wp8Dl9fX1q0aHHOcak/n+5MxTCgd6sgYkN8zI4jIiLiUlSMyWUZ1imMUD8PsgrLWbM/k1FdIn76SSIiIuKSbrrpJjIzM5k9ezZpaWn07NmT5cuXVy3If+LEiar1wwBKS0t5+OGHOXLkCH5+fowfP563336boKAgk96BnM/S07tRXtcrxuQkIiIirsdiGIZhdojLlZ+fT2BgIHl5eVqDwgR/+3wvr607yqguEbw2ra/ZcURERC6a7iEaB32dLt3RrCKufm4NNquFjX8aQaifp9mRREREGsTF3j9o3ptctql9nVuor96XQVZhmclpREREROSMJadHiw1uH6pSTERE5DxUjMll6xjhT0JsEJUOg0+2pZgdR0REREQAwzBYuv0kAJN6atF9ERGR81ExJnViap+WAHy4OYkmMDtXREREpNHbnZLPkawiPN2sjO4aaXYcERERl6RiTOrEhIRoPN2sHEgvZGdyntlxRERERJq9M9MoR3aJwM9Te26JiIicj4oxqROB3u6M7eb8TeTCLUkmpxERERFp3uwOg093np5GmaBplCIiIheiYkzqzNQ+zkX4l24/SWmF3eQ0IiIiIs3XxqPZpOeXEeDlxlWdwsyOIyIi4rJUjEmdubJdC2KCvMkvreSrvelmxxERERFpts4suj++exSebjaT04iIiLguFWNSZ6xWCzecXoR/4WZNpxQRERExQ1mlnS92pQIwUbtRioiI/CgVY1KnzuxOuf5QFim5JSanEREREWl+1u7PJL+0kogATwa0aWF2HBEREZemYkzqVGyID1e0DcEwYNGWZLPjiIiIiDQ7S3Y4p1FO6BGNzWoxOY2IiIhrUzEmde7MIvwLtyTjcBgmpxERERFpPgrLKvn69Fqvk3rGmJxGRETE9akYkzo3rnskfp5unMgpZtOxHLPjiIiIiDQbX+1Jo6zSQdtQX7rFBJgdR0RExOWpGJM65+PhxrU9ogBYuFnTKUVEREQaypLTu1FO7BmNxaJplCIiIj9FxZjUi6l9nYvwf7ErlcKySpPTiIiIiDR9WYVlrD+UBcDEBO1GKSIicjFUjEm96N0qmLZhvpRU2Pl850mz44iIiIg0eV/sSsXuMOjRMpC2YX5mxxEREWkUVIxJvbBYLNWL8Gs6pYiIiEi9OzONUovui4iIXDwVY1Jvru8dg9UCm4+f4khmodlxRERERJqspJxithw/hcUCE06v9SoiIiI/TcWY1JuIAC+u6hgGwEdbNGpMREREpL4s3eEcLXZluxaEB3iZnEZERKTxUDEm9WpqX+d0yo+3JmN3GCanEREREWl6DMNgyfYUACYlaBqliIhIbagYk3o1onM4wT7upOeX8c3BTLPjiIiIiDQ5+9IKOJBeiIfNyphukWbHERERaVRUjEm98nSzVS0A+5EW4RcRERGpc2cW3b86PoxAb3eT04iIiDQuKsak3k3t2xKAFXvTOVVUbnIaERERkabD4TD4dId2oxQREblUKsak3nWNDqRLVADldkfV+hciIiIicvm2nDhFSm4Jfp5uDI8PNzuOiIhIo6NiTBrEjadHjS3U7pQiIiIidebMLx3HdI3Ey91mchoREZHG55KKsZdeeom4uDi8vLwYMGAAmzZtuuC1e/bs4YYbbiAuLg6LxcLzzz9/zjWPPfYYFoulxiM+Pv5SoomLmtQzBg+blT0n89l7Mt/sOCIiIiKNXoXdwec7UwGY1DPa5DQiIiKNU62LsQ8++ICZM2fy6KOPsnXrVhISEhgzZgwZGRnnvb64uJi2bdvy1FNPERl54V1yunbtSmpqatVj/fr1tY0mLizY14ORXZzD+xduSTI5jYiIiEjjt/5gFqeKKwj18+DKdi3MjiMiItIo1boYmzt3LnfccQczZsygS5cuvPLKK/j4+DBv3rzzXt+vXz+effZZbr75Zjw9PS/4um5ubkRGRlY9QkNDaxtNXNzUvrEAfLIthfJKh8lpRERERBq3M9Mor+0RjZtNK6SIiIhcilp9By0vL2fLli2MHDmy+gWsVkaOHMmGDRsuK8jBgweJjo6mbdu23HrrrZw4ceKyXk9cz9AOYUQEeHKquIKVielmxxERERFptIrLK/lqr/N+aqKmUYqIiFyyWhVjWVlZ2O12IiIiahyPiIggLS3tkkMMGDCAN998k+XLl/Pyyy9z9OhRhgwZQkFBwXmvLysrIz8/v8ZDXJ/NauH63lqEX0RERORyfZ2YQXG5ndgQb3rFBpkdR0REpNFyiTHX48aNY+rUqfTo0YMxY8bwxRdfkJuby4cffnje6+fMmUNgYGDVIzY2toETy6Wa2sdZjK3Zn0F6fqnJaUREREQap6Wnp1FOSojBYrGYnEZERKTxqlUxFhoais1mIz295jS49PT0H11Yv7aCgoLo2LEjhw4dOu/5WbNmkZeXV/VIStJi7o1F2zA/+rYOxmHAoq0pZscRERERaXROFZWzZn8moN0oRURELletijEPDw/69OnDypUrq445HA5WrlzJwIED6yxUYWEhhw8fJioq6rznPT09CQgIqPGQxmNq3zPTKZMwDMPkNCIiIiKNy7LdaVQ6DDpHBdAhwt/sOCIiIo1aradSzpw5k9dee4358+eTmJjInXfeSVFRETNmzABg2rRpzJo1q+r68vJytm/fzvbt2ykvLyclJYXt27fXGA123333sXbtWo4dO8Z3333H5MmTsdls3HLLLXXwFsXVXNMjGm93G0cyi9h6ItfsOCIiIiKNypndKK/TaDEREZHL5lbbJ9x0001kZmYye/Zs0tLS6NmzJ8uXL69akP/EiRNYrdV928mTJ+nVq1fVn5977jmee+45rrrqKtasWQNAcnIyt9xyC9nZ2YSFhTF48GC+//57wsLCLvPtiSvy83RjXPdIFm1NYeHmJPq0DjY7koiIiEijcDK3hE3HcgCYkKBiTERE5HJZjCYwly0/P5/AwEDy8vI0rbKR+P5INjf/53v8PN3Y9OcR+HjUuqMVERG5bLqHaBz0dar2n28O8+QX++jfJoQPf113S5mIiIg0NRd7/+ASu1JK8zOgTQitQnwoLKtk+e40s+OIiIiINApLtp8EtOi+iIhIXVExJqawWCxM6XN6Ef7NySanEREREXF9hzIK2HMyHzerhfHdzr9JlYiIiNSOijExzQ19WmKxwIYj2ZzILjY7joiIiIhLOzNa7KqOYQT7epicRkREpGlQMSamiQnyZnD7UAA+2qpRYyIiIiIXYhhGVTE2UdMoRURE6oyKMTHVmemUH29JxuFo9PtAiIiIiNSL7Um5nMgpxtvdxqguEWbHERERaTJUjImpxnSNxN/LjZTcEjYcyTY7joiIiIhLOjNabHTXCO3mLSIiUodUjImpvNxtTExwTgf4cHOSyWlEREREXE+l3cFnO1MB7UYpIiJS11SMielu7BsLwPLdaeSVVJicRkRERMS1bDiSTVZhGcE+7gzpEGZ2HBERkSZFxZiYrkfLQDpG+FFW6eCznSfNjiMiIiLiUs5MoxzfPQp3m27fRURE6pK+s4rpLBYLU/s4R40t3KzdKUVERETOKK2ws3x3GgCTesaYnEZERKTpUTEmLuG6XjG4WS1sT8rlYHqB2XFEREREXMLqfRkUllUSHehF39bBZscRERFpclSMiUsI8/fk6vhwABZu0agxEREREaieRjmhZzRWq8XkNCIiIk2PijFxGVP7tARg0dYUKuwOk9OIiIiImCuvpIJV+zMAmJSgaZQiIiL1QcWYuIyr48MJ9fMgq7CMtfszzY4jIiIiYqov96RRXumgQ7gfnaP8zY4jIiLSJKkYE5fhbrMyuZfzt6Efbk4yOY2IiIiIuZaenkZ5Xa8YLBZNoxQREakPKsbEpUzt69ydctW+DLIKy0xOIyIiImKOjPxSvjucBcDEhGiT04iIiDRdKsYuhr0Str0DDq17Vd86RviT0DKQSofBJ9tSzI4jIiIiYorPdqbiMKB3qyBiQ3zMjiMiItJkqRi7GF/cB0vuhkV3QKVGMdW3KadHjS3cnIxhGCanEREREWl4S3Y4p1FO6qlF90VEROqTirGL0eoKsLrB7o/gnRugJNfsRE3axIRoPN2s7E8vYFdKntlxRERERBrUsawidiTlYrNaGN89yuw4IiIiTZqKsYuRcDPcuhA8/OHYOnhjHOQlm52qyQr0dmdM10jAOWpMREREpDlZenq02KD2oYT5e5qcRkREpGlTMXax2g2HGV+AXyRk7IXXR0H6HrNTNVlT+7YEYMn2FEor7CanEREREWkYhmHwyXbnOquTtOi+iIhIvVMxVhtRPeCXX0NoJyg4CfPGwpG1Zqdqkq5sF0p0oBf5pZV8tTfd7DgiIiIiDWLPyXyOZBbh6WZldNcIs+OIiIg0eSrGaisoFn7xJbQeBGX5zjXHdi40O1WTY7NamNLHOWps4eYkk9OIiIiINIwlp0eLjewcgb+Xu8lpREREmj4VY5fCOxhuWwRdJ4OjAhb9Etb/A7SDYp2a0se5O+X6Q1mczC0xOY2IiIhI/bI7jKr1xSb21DRKERGRhqBi7FK5e8EN8+CKu51//vox+OJ+cGg9rLrSqoUPA9qEYBiwaKsW4RcREZGmbdPRHNLzy/D3cmNYpzCz44iIiDQLKsYuh9UKY5+EMU8CFvjhNfhwGlRodFNdmdrXOWps4ZZkDI3IExERkSZs6Q7nNMrx3aLwdLOZnEZERKR5UDFWFwbeDVPfAJsn7PsM5k+EomyzUzUJ47tH4uth43h2MRuO6O9UREREmqaySjtf7EoDYJKmUYqIiDQYFWN1petkmPYJeAVB8iaYNxpyjpqdqtHz8XDj2h7Om8MZb/zAY0v3kJqnEXkiIiLStHxzIIu8kgrC/T0Z0LaF2XFERESaDRVjdan1lfCLryAwFrIPwX9HQcpWs1M1ejNHd6RP62DKKh28+d0xrnpmDX9avIuknGKzo4mIiIjUiTO7UU5IiMZmtZicRkREpPlQMVbXwjrBL1ZARHcoyoQ3r4WDK8xO1ahFBHjx0W8GsuCXAxjQJoRyu4N3N57g6ufWcP/CHRzNKjI7ooiIiMglKyyr5OvEdEDTKEVERBqairH6EBAFM76AtldDRRG8exNsfcvsVI2axWJhUPtQPvj1QD789UCGdAil0mGwcEsyI/6+ht+/v42D6QVmxxQRERGptRV70yitcNAm1JfuMYFmxxEREWlWVIzVF68AuHUhJNwChh2W/hZWzwHtrHjZ+rcJ4e1fDGDxXVcyIj4chwFLtp9k9PPfcNeCLew9mW92RBEREZGLtmT7SQAmJkRjsWgapYiISENSMVafbO5w3csw9H7nn9c+BUvuAXuFubmaiF6tgvnv9H589tvBjO0aiWHAF7vSGP/Pdfxy/mZ2JueaHVFERETkR2UXlrHuYBagaZQiIiJmUDFW3ywWGP4wXPsPsFhh+zvw3s1QVmh2siajW0wgr/ysD1/eO5QJCdFYLPB1YjoT//Utt8/bxJbjOWZHFBERETmvL3alYncY9GgZSNswP7PjiIiINDsqxhpK35/Dze+Cmzcc+hreHA8F6WanalI6Rfrz4i29+HrmVVzfOwab1cLaA5nc8PIG/u+179lwOBtDU1lFRETEhZw9jVJEREQanoqxhtRpHEz/HHxCIXUH/HckZB00O1WT0y7Mj7k39mT1H4dxc79Y3G0WvjuczS2vfc+Nr25g7YFMFWQiIiJiuqScYjYfP4XFAhNUjImIiJhCxVhDa9kHfvEVhLSF3BPw31Fw4nuzUzVJrVr48NQNPVhz/9X87IrWeLhZ+eHYKW6ft4nr/v0dX+9NV0EmIiIipvl0p3O02MC2LYgI8DI5jYiISPOkYswMLdrBL1ZATB8oOQVvTYK9S81O1WTFBHnzxHXdWPfA1fxicBu83K3sSMrll29t5pp/rmfZrlQcDhVkIiIi0rCWnp5GqUX3RUREzKNizCy+oXD7Z9BxHFSWwofTYOOrZqdq0iICvHjk2i6sf3A4v7mqHb4eNvam5nPngq2MfeEblmxPwa6CTERERBrAvrR89qUV4GGzMrZrlNlxREREmi0VY2by8IGb3oE+MwADlj0AXz0CDofZyZq0UD9PHhoXz/oHh/O74e3x93LjQHohv39/O6PmruWjLclU2PU1EBGR5umll14iLi4OLy8vBgwYwKZNmy54bUVFBY8//jjt2rXDy8uLhIQEli9fXuOaOXPm0K9fP/z9/QkPD+e6665j//799f02XN6ZRfeHdQoj0Mfd5DQiIiLNl4oxs9nc4Np/wIjZzj9/909YdAdUlpmbqxkI9vVg5uhOrH9wOH8c1ZEgH3eOZBVx38IdDP/7Gt7bdILyShVkIiLSfHzwwQfMnDmTRx99lK1bt5KQkMCYMWPIyMg47/UPP/wwr776Ki+++CJ79+7lN7/5DZMnT2bbtm1V16xdu5a7776b77//nhUrVlBRUcHo0aMpKipqqLflchwO46xplDEmpxEREWneLEYTWH08Pz+fwMBA8vLyCAgIMDvOpdvxPiy5GxyVEDfEOZrMO8jsVM1GYVkl73x/nNfXHSGrsByAqEAvfnNVO27qF4uXu83khCIiUteazD1EHRkwYAD9+vXjX//6FwAOh4PY2Fh++9vf8tBDD51zfXR0NH/+85+5++67q47dcMMNeHt7884775z3c2RmZhIeHs7atWsZOnToReVqal+nzcdymPLKBnw9bGx5ZJTuMUREROrBxd4/aMSYK0m4GW5dCB7+cGwdvDEO8pLNTtVs+Hm68Zur2rHugeE8cm0Xwv09Sc0r5dGlexjyzGpeX3eE4vJKs2OKiIjUi/LycrZs2cLIkSOrjlmtVkaOHMmGDRvO+5yysjK8vGrupujt7c369esv+Hny8vIACAkJueA1ZWVl5Ofn13g0JWemUY7pFqlSTERExGQqxlxNu+Ew4wvwi4SMvfD6KEjfY3aqZsXbw8YvBrfhmweu5onruhET5E1mQRl//TyRIU+v5t9rDlFYpoJMRESalqysLOx2OxERETWOR0REkJaWdt7njBkzhrlz53Lw4EEcDgcrVqxg0aJFpKamnvd6h8PBvffey6BBg+jWrdsFs8yZM4fAwMCqR2xs7KW/MRdTYXfw+S7n34+mUYqIiJhPxZgriuoBv/waQjtBwUmYNxaOrDU7VbPj5W7jZ1e0ZvV9w3j6hu60CvEhu6icZ5bvZ9BTq3jh64PklVSYHVNERMQ0L7zwAh06dCA+Ph4PDw/uueceZsyYgdV6/lvMu+++m927d/P+++//6OvOmjWLvLy8qkdSUlJ9xDfF+kNZ5BSV08LXg0HtWpgdR0REpNlTMeaqgmLhF19C60FQlg/v3AA7F5qdqlnycLNyU79WrPrjVcy9MYG2Yb7klVTwj68PMPipVTz35X5yisrNjikiInJZQkNDsdlspKen1zienp5OZGTkeZ8TFhbGJ598QlFREcePH2ffvn34+fnRtm3bc6695557+Oyzz1i9ejUtW7b80Syenp4EBATUeDQVZxbdv7ZHFG423YqLiIiYTd+NXZl3MNy2CLpOBkcFLPolrP8HNP79EholN5uV63u3ZMUfruLFW3rRKcKfgrJK/rX6EIOfXsWcLxLJLNBuoiIi0jh5eHjQp08fVq5cWXXM4XCwcuVKBg4c+KPP9fLyIiYmhsrKSj7++GMmTZpUdc4wDO655x4WL17MqlWraNOmTb29B1dXUm7nyz3OaakTNY1SRETEJagYc3XuXnDDPLji9G5PXz8GX9wPDrupsZozm9XChIRolv1+CK/c1oeu0QEUl9t59ZsjDH56FY8t3UNaXqnZMUVERGpt5syZvPbaa8yfP5/ExETuvPNOioqKmDFjBgDTpk1j1qxZVddv3LiRRYsWceTIEdatW8fYsWNxOBw88MADVdfcfffdvPPOO7z77rv4+/uTlpZGWloaJSUlDf7+zPZ1YjrF5XZaBnvTu1WQ2XFEREQEcDM7gFwEqxXGPgmBMfDln+GH16AgFW54Hdy9zU7XbFmtFsZ2i2RM1whW78/gnysPsT0plze/O8a7G08wtW9L7hzWjpbBPmZHFRERuSg33XQTmZmZzJ49m7S0NHr27Mny5curFuQ/ceJEjfXDSktLefjhhzly5Ah+fn6MHz+et99+m6CgoKprXn75ZQCGDRtW43O98cYbTJ8+vb7fkks5sxvlpJ7RWCwWk9OIiIgIgMUwGv+8vPz8fAIDA8nLy2tSa1Cc157FsOjXYC+Dlv3hlvfBVwu3ugLDMPj2UDb/XHWQTUf/v737jq+yPv8//jrZOyGbhEzCXmGDA1BAREVUXNR+GWpbW/FXSlG0UsVRwI2r1loFRylqVbCCKILgYkPYK6xAIBOyd879++MOJwQChJU74/18PO4H5Jz73OfKOVE+uc51XZ9jALg42bitRyR/GJRAbLC3xRGKiMipmtUaohFrCu9TTlEZvf/2HeWVBkv+NIA2Yb5WhyQiItKk1XX9oFbKxqbTrTBmPngEwOE18N51cGy/1VEJYLPZuKpNMJ/8rj8f/7YfVyUEU2E3+GTdYa59aTl/+jiJ5IwCq8MUERERC3y9NY3ySoMOLf2UFBMREWlAlBhrjGKugPu+Bf8oyE6Gd4dC6garo5KT9I0P4qP7+/LZ76/gmnYh2A34YmMqQ19ZwYNzN7AzLc/qEEVERKQeLUhKBcw2ShEREWk4lBhrrELawX1LIKwLFGbCnJtg97dWRyWn6BnTgtnj+/C/CVdxXccwDAMWbj7K9bN+5LcfrGPL4VyrQxQREZHL7GhuMaurxiyM6KbEmIiISEOixFhj5tcSxi+C+GugvBD+czds+MDqqKQWXVr5888xvfj6j1dzY9eW2Gzw7fZ0RrzxE+Nmr2H9weNWhygiIiKXyVebjmIY0Cc2kMgAbZwkIiLSkCgx1th5+ME9n0K30WBUwpcPwfczoPHvqdAkdWjpx5u/6sGSPw3ktu6RODvZWL4rk1Fv/cI9/1rFqn3ZVocoIiIil9iCTWYb5c1qoxQREWlwlBhrCpxd4Za3YMDD5tcrZsKCCVBZbm1cckYJoT68fFciy/48kLt6ReHiZOPn5Gzu/ucq7vzHSn7ck0kT2DBWRESk2UvOKGBrah4uTjZu6NLS6nBERETkFEqMNRU2G1w7FW56BWxOkPSR2VpZql0QG7KYIG+eu70ryx8exK/7RePm7MSaA8f4v3fXcOvff2HZznQlyERERBqxL6uG7g9oG0Kgt5vF0YiIiMiplBhranrdC3fPBRdPSP4O5twA+elWRyXn0KqFF8/e0oUfHrmG8VfG4u7iRNKhHO6ds44Rb/zE4q1p2O1KkImIiDQmhmGwYNMRQLtRioiINFQXlBh78803iY2NxcPDg759+7JmzZoznrtt2zZGjRpFbGwsNpuNWbNmXfQ15RzaDYdxC8ErGI5ugneHQNYeq6OSOgj39+DJEZ34acq1/G5APF5uzmxNzeOBj9Yz/NUf+XLTESqVIBMREWkUNh3O5WB2EZ6uzgzpEGZ1OCIiIlKL806Mffzxx0yaNIknn3ySDRs20K1bN4YNG0ZGRkat5xcVFREfH8/MmTMJDw+/JNeUOmjVE+77FgLjIScF3h0KKausjkrqKMTXncdu6MBPU65lwjUJ+Lq7sCs9n//3n40MfWUFn60/TEWl3eowRURE5CwWVLVRDu0Yhre7i8XRiIiISG1sxnkOMOrbty+9e/fmjTfeAMButxMVFcVDDz3Eo48+etbHxsbGMnHiRCZOnHjJrgmQl5eHv78/ubm5+Pn5nc+30/QVZsHcOyF1Pbh4wG3vQMebrY5KzlNucTnv/3KAd3/aT26xualCdKAXfxjUmtt6tMLNRV3RIiIXQmuIxqExvk+VdoO+05eSVVDKu2N7MVgVYyIiIvWqruuH8/ptuqysjPXr1zNkyJDqCzg5MWTIEFauXHlBgV7INUtLS8nLy6txyBl4B8PYr6DtcKgogU/GwOq3rY5KzpO/pyv/b3Abfn70WqZc354gbzdSjhXx6OdbuObF5Xy48gAl5ZVWhykiIiJVVu7NJquglAAvV65uE2J1OCIiInIG55UYy8rKorKykrCwmp94hYWFkZaWdkEBXMg1Z8yYgb+/v+OIioq6oOduNty84K6PoOd4wICvH4Fv/wp2teI1Nj7uLvx+UGt+nHINU2/sQKivO6k5xfx1wTYGPP897/60n+IyJchERESsdqKN8oYuLVXZLSIi0oA1yn+lH3vsMXJzcx3HoUOHrA6p4XN2gZtegcFPmF//8hp8fj9UlFobl1wQLzcX7r86nh8euYanR3Yiwt+DjPxSnvlqO1c/v4x/rNhLQWmF1WGKiIg0SyXllSzean7AO7KbdqMUERFpyM4rMRYcHIyzszPp6ek1bk9PTz/jYP3LcU13d3f8/PxqHFIHNhtc/We49W1wcoGtn8FHo6A4x+rI5AJ5uDozpn8syx++hhm3dSEq0JOsgjJmfr2Tq55bxmtL9zhmkomIiEj9WL4rg/zSClr6e9A7NtDqcEREROQszisx5ubmRs+ePVm6dKnjNrvdztKlS+nfv/8FBXA5rinn0O1uuOdTcPOFAz/C7OGQe9jqqOQiuLk4MbpPNMv+PIgX7+hGfLA3OUXlvLxkN1fNXMZL3+7ieGGZ1WGKiIg0CwuSjgBwc7cInJxsFkcjIiIiZ3PerZSTJk3inXfe4f3332fHjh38/ve/p7CwkPHjxwMwZswYHnvsMcf5ZWVlJCUlkZSURFlZGampqSQlJZGcnFzna8pl0PpaGL8IfMIhYzv8ayikb7M6KrlIrs5O3N6zFUsmDeS10d1pG+ZDfmkFry9L5qrnljHj6x1kFah9VkRE5HLJKyln6c4MAEYmRlocjYiIiJyLy/k+4K677iIzM5MnnniCtLQ0EhMTWbx4sWN4fkpKCk5O1fm2I0eO0L17d8fXL774Ii+++CIDBw5k+fLldbqmXCYtu8L935ntlFm74L3rzSH98QOtjkwukrOTjZu7RXBTl5Z8uz2N15Yms/1oHm+v2Mf7vxzgV31i+N3AeML8PKwOVUREpEn5ZmsaZRV22oT60KGlr9XhiIiIyDnYDMMwrA7iYuXl5eHv709ubq7mjV2I4uMw7x44+DM4ucItb0HXO6yOSi4hwzBYtjOD15Yls+lQDmC2X97VK4oHBrUmMsDT2gBFRCyiNUTj0Jjep/97dzU/7sli8nVtmXBtG6vDERERabbqun5olLtSyiXm2QJ+/Tl0uhXs5eZulT+9Ao0/ZypVbDYbgzuEMf8PV/DBvX3oHduCsgo7H646yKAXvufRzzZzMLvQ6jBFREQatYz8En5OzgLg5m5qoxQREWkMlBgTk6sHjHoP+j1ofv3dNFj0MNgrLQ1LLi2bzcaAtiF88rv+/Oc3/biidRDllQbz1h7i2pdWMOnjJPZmFlgdpoiISKO0cPNR7AZ0jw4gOsjL6nBERESkDpQYk2pOTnD9dBg2HbDB2nfgkzFQXmx1ZHKJ2Ww2+rcOYu5v+vHZ7/szqF0IlXaDzzemMuTlFUyYu4FdaflWhykiItKonNiNcmS3CIsjERERkbpSYkxO1/9BuGM2OLvDzq/g/ZuhMNvqqOQy6RkTyJzxffhywpUM7RiGYcBXm48ybNYP/O7DdWxNzbU6RBERkQbvYHYhSYdycLLBjV2VGBMREWkslBiT2nW6FcbMB48AOLwG3rsOju23Oiq5jLq2CuCdMb1Y9P+u5sYuLbHZ4Jtt6dz0+k/cO2ctC5JSOXSsiCawX4eIiMgl92VVtdiVCcGE+LpbHI2IiIjUlYvVAUgDFnMF3PctfDQKspPh3aHwq08gsofVkcll1DHCjzfv6cGe9Hze/D6ZLzcdYdnODJbtzAAg2MedHtEB9IhpQY/oFnRt5Y+Hq7PFUYuIiFjHMAzmJ6UCMDJRQ/dFREQaE5vRBMo/GtMW3o1S3lH49x2QvgVcveGOOdD2OqujknqyP6uQuasPsvbAcbYdyaW8sub/MlycbHSK8KN7dIuqZFkAkQGe2Gw2iyIWEak7rSEah4b+Pm1NzeWm13/CzcWJ9VOH4OvhanVIIiIizV5d1w+qGJNz82sJ4xeZg/j3fQ//uRtGzIIeY6yOTOpBXLA3j9/YEYCS8kq2Hcll/cHjbDiYw4aU42Tkl7LpcC6bDucy55cDAIT6utMjugU9YgLoEd2CzpGqKhMRkabry01mG+WQDqFKiomIiDQySoxJ3Xj4wT2fwpcPwab/mH/mpsKgR0GVQc2Gh6szPWMC6RkTCJitI6k5xWxIyWHDweNsTDnOtiN5ZOSXsnhbGou3pQHg6myjU4R/jWRZRICnld+KiIjIJWG3G475Yjd3UxuliIhIY6PEmNSdsyvc8hb4t4IfXoAVMyH3sFk95qxPR5sjm81GqxZetGrhxc1VW9OXlFey+XAuG1KOs+HgcTak5JBVUErSoRySDuXw3s/mY8P9PBxJsu7RLegc6Ye7i6rKRESkcVlz4BhpeSX4ergwqF2I1eGIiIjIeVJiTM6PzQbXTgW/CFj4Z0j6CArS4I73wd3H6uikAfBwdaZPXCB94qqryg4fL3YkytanHGfH0XzS8kpYtCWNRVvMqjI3Zyc6R/pVVZWZg/3D/T2s/FZERETOaUFVtdjwzuEaGyAiItIIKTEmF6bXveDbEj4dD8nfwZwb4Fefgm+Y1ZFJA2Oz2YgK9CIq0MuxU1dRWcVJVWU5bEw5TnZhmdmSmZIDP+0HIMLfg+4xLehZlSzr2NIPNxcnC78bERGRamUVdhZtOQpoN0oREZHGSokxuXDthsO4hTD3Tji6Cd4dAr/+HILbWB2ZNHBebi70iw+iX3wQYFaVpRwrMof6VyXLdqblcSS3hCObj7Jws/lLh7uLE10i/R27X/aIbkGon6rKRETEGj/sziS3uJwQX3fHv2kiIiLSuCgxJhenVU+471v49+1wbB+8OxRGz4PoflZHJo2IzWYjJsibmCBvbuvRCoDC0go2Hc5hY9Vg/w0pxzleVM66g8dZd/C447GtWnia7ZfRAfSIaUGHln64OquqTERELr8FVbtRjugagbOTNiMSERFpjJQYk4sX1BruW2JWjqWuhw9Gwm3vQMebrY5MGjFvdxeuaB3MFa2DAbOqbH9WYVW7pTmvbHd6PoePF3P4eDFfVv1y4uHqRNfIALpXDfbvEd2CEF93K78VERFpggpLK1iy3ZyTeUv3CIujERERkQulxJhcGt7BMPYr+O+9sPtr+GQMDH8O+v7O6sjkYhUdg+P7ISgBPPwtC8NmsxEf4kN8iA+39zSrygpKK9h0KMcx1H9jSg65xeWsOXCMNQeOOR4bHejlqCjrEd2C9uG+uKiqTERELsKS7emUlNuJC/amS6R1/z6KiIjIxVFiTC4dNy+46yNYNBnWz4avH4HcwzDkKXBSEqLBqyiDrN2Qvg0ytpl/pm+HfLMSCydXiL0K2t8I7W4Af+uHDPu4u3BlQjBXJphVZXa7wb6sQjakHGdj1ayy3Rn5pBwrIuVYEfOrdg7zdHWmaytzVlnP6BZ0jw4gyEdVZSIiUncLklIBuLlbBDab2ihFREQaK5thGIbVQVysvLw8/P39yc3Nxc/Pz+pwxDDgp5dh6dPm151HwS1vgYsSDw2CYZgJy4ztkL7VTH6lb4PsPWCvqP0xXkFQlF3ztoju0O5GaH8DhHaEBvpLQV5JOUkn2i9TzB0w80tO/z5jg7zoEd2C7lWD/duFqapMpDnQGqJxaGjvU3ZBKX2mL6XSbrD0zwNpHeJjdUgiIiJyirquH1QxJpeezQZX/xn8ImHBg7D1MyjIMKvJPAOsjq55KcmFjB1V1V/bqpJh26E0t/bz3f0hrBOEdTT/DO0EoR3Aww+ykmHXQti5CA6thiMbzeP7Z6FFbFWS7EaI6gvODed/LX4ergxoG8KAtiGAWVW2N7PAsfvl+pTjJGcUcCC7iAPZRXy+0awA8HJzplurAHrGtKBHTADdo1rQwtvNym9FREQaiEVb06i0G3SJ9FdSTEREpJFTxZhcXnuXwcdjoCzfrCq651Pwb2V1VE1PZQVkJ5sVYBnbq9sgc1NqP9/JBYLbViW/OkJYZzMZ5hdZt8qvggzYvRh2LoS930NlafV9noHQ9nozSdb6WrPFtoHLLSpn46HqirKklBzyS0+vKosP9qZ7tJko6xHdgrZhvtqFTKSR0xqicWho79Md//iFtQeOM/XGDtx/dbzV4YiINAp2u52ysjKrw5AmxNXVFWdn5zPeX9f1gxJjcvkd3Qz/vgMK0sA3An79XzMhI+fPMCA/reYMsPRtkLULKs/wj4xfZFXyq1P1EdQGXC5R9VNZoZkA3bnQTJYVH6++z8XDTI61uwHaDTc3aWgEKu0GyRlmVdn6g8fZkHKcfZmFp53n4+5Ctyh/c05ZTAt6RLXA38vVgohF5EJpDdE4NKT36fDxIq567ntsNlj56GDC/T0sjUdEpDEoKytj//792O12q0ORJiYgIIDw8PBa530qMSYNS84h+GiUmcBx9zPbKuMHWh1Vw1ZaAJk7T2mD3Foz8XQyN5+qBFhVBdiJv3u2qL+YKysgZSXsWmQmynIOVt9nczLbLNvdYFaTBbWuv7gugZyiMjY6ZpWZVWWFZZWnndc6xJse0S0cO2C2CfXBSVVlIg2W1hCNQ0N6n95avpfnFu+kf3wQ//ltP0tjERFpDAzDICUlhfLyciIiInDSxmxyCRiGQVFRERkZGQQEBNCyZcvTzlFiTBqe4uMw7x44+LO5w+Etb0HXO6yOynr2Sji276TkV1Ui7PgBoJb/PG1OEJRQPQPsxEww/+iGtfunYZjfx65FsPMrOLqp5v0h7at2uLzRHOTfkGKvg0q7wa60fEeibGNKDvuzTq8q8/VwITEqwJEsS4wKwN9TVWUidWEYBoePF5NVUEr36MuT5NcaonFoSO/T9bN+YGdaPjNv68LdfaItjUVEpDEoLy8nOTmZiIgI/P39rQ5Hmpjs7GwyMjJo27btaW2VSoxJw1ReAvMfgG1fmF8PmQZXTmywOxpecgWZp7RBboXMXVBRXPv5PmGnt0EGtwPXRti2kXsYdn1tJskO/FRzB0yfcHN3y3Y3QtzVjXYH02OFZWysSpStP3icTYdyKS6vWVVms0FCiA89ols4BvvHB6uqrMEpL4bUDXBolfmnfyvoONKsenQ68xwDuXCVdoP9WYVsO5LL1tRctqbmse1ILnklFcQFe/P95EGX5Xm1hmgcGsr7tCstn2GzfsDV2ca6x4eqfV5EpA5KSkrYv38/sbGxeHp6Wh2ONDHFxcUcOHCAuLg4PDxq/p6sXSmlYXL1gFHvmbPGVr0J302D3FQY/lzT+mWzvLiqDbKqAuxEMqwws/bzXTzN3R9rtEF2ajQzuerEvxX0+Y15FOfAniXmLpd7vjPnz617zzzcfKHNEDNJ1mZoo9rJNNDbjcEdwhjcIQyAiko7O9Pyq5JlZhvmwewi9mQUsCejgI/XHQLAz8PFHOpfNdg/MSoAXw/9slWvCrMgZZXZCnxoNRxJAnt5zXNW/8NMVncYYSbJoq9oUDuwNibllXaSMwrYmprLtiN5bE3NZfvRPIpqaU92dbbh4+5CeaUdV+fGVVkqTc+CJHPn4kHtQpUUExE5T7XNgBK5WJfi50oVY2KdlW/CN48DBrS/CUb9C1wb2ScIdjvkHDATYCdmgKVvh2N7wahtsKQNAuNOaYPsBC1im1Zi8HxUlML+H80k2c5FZpLsBCcXiL3KTJK1v6FJ7GiaVVDKxpQcx1D/zYdzKCmv+bNis0HbUF/H7pc9YloQF+StqrJLxTDMXVxTVkLKavPPY3tPP88nHKL7QmRPyNhp/oyW5Fbf7xVk/r+r40iIGwDO+iW5NiXllexKy2frkeoqsJ1p+ZRVnP7/SA9XJzq29KNzpD+dI/zpFOlHm1Bf3FwuX0JMa4jGoSG8T4ZhcNVz35OaU8wbv+rOTV0jLIlDRKSxOVExVltFj8jFOtvPl1oppXHY9gV8/juoLIVWfWD0PPAOsjqq2hUdO30OWMYOKD99rhQAnoFVia/OZiVYaCcIbQ9u3vUbd2Nit8ORjWa75a5FZtXdyVp2MxMR7W4wX9sm8KlTeaWdnUerZ5VtSDnOoWOnt9Z6uDqREOpDQogPbcJ8aR3iQ5swH2ICvXBRFc3ZVZSaFWCHVpmJsEOroCj79PNCO5qtktH9zCMgpubPWEUZ7P8Bts83f0ZP3gjDI6A6SRY/6NLt+trIFJZWsOOoWQG2taoSbE9GAZX205cavu4udIyoSoJF+tE5wp/4EB+c6zkBrDVE49AQ3qf1B48x6q2VeLs5s27qUDzdmukHWiIi50mJMYiNjWXixIlMnDjR6lCaHCXGqjSExZJchIO/wH9GQ0kOBLaGX39mVlVZpaIUsnafshvkNsg/Wvv5zm7mIPkT1V8n2iB9wppE4sZS2XvN3S13LTLb3E7ejCAgpmp4/w0Q3b9JtbRl5Jew4WCOY17Z5sO5lNZSXQNmm1lcsDdtQn3NxFmomTCLC/bG3aWZ/tJWdAwOralui0zdYCbfT+biYVaCRfU1f36iep/fDq6V5easvO0LYMf/oCir+j53f2g3HDreDK0HN86ZgHWQW1RuzgOrqgTbeiSX/VmF1LaqaOHlWpUAMyvBOkf6EdXCq0FUQWoN0Tg0hPfpiQVb+WDlQW7rHsnLdyVaEoOISGPUWBNjgwYNIjExkVmzZl30tTIzM/H29sbLy+viA5MalBir0hAWS3KRMnfBR6Mg9xB4h8CvPoHIHpf3OQ3DfL707TUH4mfvqTkY/mQB0SfNAKuaBxbYukklZRqsgkzYvdhMku1dBhUl1fd5toC215uJstbXNrmqvIpKOynHikiumk2WfNJx6nD/E5xsEBPkXZ0sC/WhTagvrUO98XJrQj+vhgHH91fNB1tlJsJOrTQE8AqurgSL6mdWH16qqi57pZmE274Atn9Zsx3YzQfaDjMryRKGglvjXAxl5pey7Uj1PLCtR3JrrWwECPfzoFOEH50i/elcVRHW0t+jwc4V0RqicbD6fSqvtNNv+lKyC8uYM743g9qF1nsMIiKNVVNNjBmGQWVlJS4uTWhtfYHKyspwc7OmY0KJsSpWL5bkEsk7Cv++A9K3gKs33DEH2l53aa5dkntSAmx7dRtkaW7t53v4nzQD7EQbZAfw0M9Xg1BWCHu/N6vJdi+G4mPV97l4mK1s7W+EtsPBJ8SyMC83u90gNaeY5MwCktMLqhJn+ezJKCC/5AzJXSAywJM2YSfaMn1IqKo28/dsBDOyKsvh6OaqtsiqozDj9POC2lQnwqL7Q2B8/VRw2u1weE1VkmwB5KVW3+fqZW4o0XEktBkG7j6XP57zZBgGR3NLHK2Q26qSYOl5pbWeHxXoWVUB5m8mwyL8CfFtXLvKag3ROFj9Pi3flcG42WsJ8nZj1V8GayMIEZHz0BgTY+PGjeP999+vcdvs2bMZP348ixYtYurUqWzZsoVvv/2WqKgoJk2axKpVqygsLKRDhw7MmDGDIUOGOB57aiulzWbjnXfeYeHChXzzzTdERkby0ksvcfPNN58ztsrKSn7729+ybNky0tLSiI6O5g9/+AN//OMfa5z33nvv8dJLL5GcnExgYCCjRo3ijTfeACAnJ4cpU6Ywf/58cnNzSUhIYObMmdx0001MmzaN+fPnk5SU5LjWrFmzmDVrFgcOHHC8Pjk5OfTu3Zs333wTd3d39u/fz4cffsirr77Krl278Pb25tprr2XWrFmEhlZ/oLRt2zamTJnCDz/8gGEYJCYmMmfOHFJTUxk8eDCHDh0iPDzccf7EiRNZv349P/74Y62vx6VIjCm1KQ2HX0sYvwg+GQP7vof/3A0jZkGPMXW/RmW5OVT71DbI3EO1n+/kAsHtqpJfHavngflFqg2yIXPzhg43mUdlhVkltHOhORz9+AEzWbZ7MWAzW+Xa32AO8A9OsDryS8rJyUZUoBdRgV5cc1L1gmEYZOaXmrtfpueTnFnAnvQC9mYWkFVQRmpOMak5xSzfVXOX1FBfd0d1WUKYryNxFuTtZl21T0kuHFpbnQg7vA4qTqlUcnaDiO4ntUX2tW5WoZNTdULuur/BkQ3mTLLtCyAnpTph5uIBCUOgw83Q7nozGV/PDMMg5ViRow3yxA6RxwrLTjvXZoO4YG9HG2TnCH86RfhrVz5pNr5MOgLAjV1bKikmInKRDMM4Y9fD5ebp6lynde2rr77K7t276dy5M08//TRgJnQAHn30UV588UXi4+Np0aIFhw4d4oYbbuBvf/sb7u7ufPDBB4wYMYJdu3YRHR19xud46qmneP7553nhhRd4/fXXueeeezh48CCBgYFnjc1ut9OqVSs+/fRTgoKC+OWXX/jtb39Ly5YtufPOOwF46623mDRpEjNnzmT48OHk5uby888/Ox4/fPhw8vPz+eijj2jdujXbt2/H2fn8xrAsXboUPz8/lixZ4ritvLycZ555hnbt2pGRkcGkSZMYN24cixYtAiA1NZUBAwYwaNAgli1bhp+fHz///DMVFRUMGDCA+Ph4PvzwQx5++GHH9f7973/z/PPPn1ds50uJMWlYPPzgnk/hy4dg03/MP3NTYdCjNRNVhmHO/Eqv2gkyY7v596xdUHn6L3WAmew6eQZYWCezqqSZDsluMpxdIPZK8xj2N/NnYeciM0l2ZKOZUDm0CpY8YSZB299gDkmP6GEmMZogm81GqJ8HoX4eXJkQXOO+Y4VljjbMPRn5jr8fzS0hI7+UjPxSftlbczB9gJermSyrqixrUzXHLNzvErfHnWhvPrktMn0bNWbLgdk6e2JIflQ/MynWEOd4OTlBq17mMfQZOLqpKjE2H47tMwf47/zKTOzFX2NWkrUbDl5nXwxdiEq7wb7Mgup5YKm5bD+SR37p6ZWFzk422oT6VM0DM1shO7T0w9tdSwZpnorLKvlmm9kiPTJRO1GKiFys4vJKOj7xjSXPvf3pYXUaK+Lv74+bmxteXl6O6qWdO81xHU8//TRDhw51nBsYGEi3bt0cXz/zzDN88cUXfPnll0yYMOGMzzFu3DhGjx4NwPTp03nttddYs2YN119//Vljc3V15amnnnJ8HRcXx8qVK/nkk08cibFnn32WP//5zzWqyHr37g3Ad999x5o1a9ixYwdt27YFID4+/pyvyam8vb3517/+VaOF8t5773X8PT4+ntdee43evXtTUFCAj48Pb775Jv7+/sybNw9XV/MD1hMxANx3333Mnj3bkRj73//+R0lJieP7uly0ypWGx9kVbnkL/FvBDy/AipnmL8tRfU5qg9xWc0e4k7n51JwBduLv5zNYWxonm6066TnwYTOpumuRWU124EczcfrTLvjpFfAJNyt12t8EcQPApXG1f12oQG83+sQF0ieuZvIlv6ScvZmFZoWZI3FWwKHjReQUlbP2wHHWHqj535yPuwutT1SYnfRnqxZeddtZsLLCTGwfWl2dDMs/cvp5LeLMSrDovmYiLLht40tq2mwQkWgeg58w/z92IkmWtRv2fGMeTi4QN9BMkrW/6YIq38oq7OxOz2f7kepKsB1H82v9ZNbNxYn24b50OqkSrF24Lx6uzXTjBpFaLN2ZTmFZJa1aeNIjWmsJEZHmrlevXjW+LigoYNq0aSxcuJCjR49SUVFBcXExKSkpZ71O165dHX/39vbGz8+PjIxaRoTU4s033+S9994jJSWF4uJiysrKSExMBCAjI4MjR44wePDgWh+blJREq1ataiSkLkSXLl1Omyu2fv16pk2bxqZNmzh+/Dh2u7mBWEpKCh07diQpKYmrr77akRQ71bhx45g6dSqrVq2iX79+zJkzhzvvvBNv78s7Q1qJMWmYbDa4dir4RcDCP0PSv82jxjlOZsXXiRlgJ+aB+Uc3vl+a5fLwj4Q+vzGP4hxI/s5Mku1ZYg5IXz/HPNx8IGGwmYhoM7RZJlF9PVxJjAogMSqgxu3FZZXsy6pKlJ00x+xgdhEFpRVsOpTDpkM5NR7j7uJE6xCfGtVlCaE+xPgauB5ZX5UIW2m2RZYV1AzEycUcjB/VrzoR5ht2eb/5+mazQXhn87j2ccjYWbW75ZdmonDvUvP46k8Qe5WZJOswAnxOH/ZdUl7JjqN5NeaB7U4roKzy9F1Mvdyc6djSzzEPrHOkPwmhPmoLEzmHBVVtlCMTIxrsJhIiIo2Jp6sz258eZtlzX6xTkzSTJ09myZIlvPjiiyQkJODp6cntt99OWdkZOpmqnJocstlsjkTS2cybN4/Jkyfz0ksv0b9/f3x9fXnhhRdYvXo1AJ6enmd9/Lnud3Jy4tRR9OXl5aedd+rrUFhYyLBhwxg2bBj//ve/CQkJISUlhWHDhjlei3M9d2hoKCNGjGD27NnExcXx9ddfs3z58rM+5lJQYkwatl73mi2QP7xgJi9OVAOFdTLb4hpi+5Q0TJ4B0OV286goNSvIdi4yK8ryj1bPfnJygZgrzeH97W6AgCirI7eUp5sznarmSZ2srMLOwexCxy6ZJ+aZ7csqpLTCzvajeWQfPUCl0258nHYR6bQLmy0FbDX/sa909YWoPjjH9DdbIyN7NtqdGy9YaHvzGDQFspJhR9XP4tFNsH+FeSz8MxVR/Tgcfh2r3K9gzTEPtqXmkZxZQKX99D10/DxcqqvAIs33Ly7Yu26VfCLikFtUzvJd5qf3IxMjLY5GRKRpsNlsjWKXdDc3Nyorzz0L7eeff2bcuHHceuutgFlBdmJI/eXw888/c8UVV/CHP/zBcdvevXsdf/f19SU2NpalS5dyzTXXnPb4rl27cvjwYXbv3l1r1VhISAhpaWkYhuH4QOjkQfxnsnPnTrKzs5k5cyZRUebvUOvWrTvtud9//33Ky8vPWDV2//33M3r0aFq1akXr1q258sorz/ncF6vh/zSKtB1mHiKXiou7Ofg8YQjc8CIc3WgmyXYuhMwd1cmIrx+B8K5mkqz9jWZrrqoFALMFr02YL23CfM0b7HbI3IH94G6Kkn/G+fBqPItST3vcYSOYdfa2rLO3Y529HbtLWsEOJ6LSvWhzwIOE0BRHpVnrUB98mttcq+AEuPrPHO/xEMm7tlKxbT4tU78htmQnLodWEntoJbFAgr0tX1f2Id/eh1LvCHMeWFUrZOdIf1q18FRli8gl8PXWo5RXGrQP96Xtif/fiYhIsxAbG8vq1as5cOAAPj4+Z6zmatOmDZ9//jkjRozAZrPx17/+tU6VXxeqTZs2fPDBB3zzzTfExcXx4YcfsnbtWuLi4hznTJs2jQceeIDQ0FDHoP2ff/6Zhx56iIEDBzJgwABGjRrFyy+/TEJCAjt37sRms3H99dczaNAgMjMzef7557n99ttZvHgxX3/99Tl3hY6OjsbNzY3XX3+dBx54gK1bt/LMM8/UOGfChAm8/vrr3H333Tz22GP4+/uzatUq+vTpQ7t27QAYNmwYfn5+PPvss46NDy63ZvYbh4jIKZyczCqlyJ4w+K+QvbdqLtkic2h/2mbzWD7DbNNtf6M5wD/6CnPwf3NVVgSp66t3izy0FkpzcQJ8TpxjczKTidH9MaL6khaQyN5iP7LS86nIKMAnowDfjAJyi8s5mF3EwewivttRc65ChL9HjR0yT8wxC/BqOptmZOSVsO2IORD/xHD81JwTO2/2BfoSSSbXO69lpNtauhq76OW0m15Ou/mr60cY4T2xtb3ZbLkMbGnltyLS5FS3UapaTESkuZk8eTJjx46lY8eOFBcXM3v27FrPe/nll7n33nu54oorCA4OZsqUKeTl5V22uH73u9+xceNG7rrrLmw2G6NHj+YPf/gDX3/9teOcsWPHUlJSwiuvvMLkyZMJDg7m9ttvd9z/2WefMXnyZEaPHk1hYSEJCQnMnDkTgA4dOvD3v/+d6dOn88wzzzBq1CgmT57MP//5z7PGFRISwpw5c/jLX/7Ca6+9Ro8ePXjxxRe5+eabHecEBQWxbNkyHn74YQYOHIizszOJiYk1qsKcnJwYN24c06dPZ8yYMZfqZTsrm3Fq82gjlJeXh7+/P7m5uefMYoqI1FlhFuxebCbJ9i6DiuLq+zwCoO31ZpKs9WBw9znjZZqEgoyTdotcZbb52U/Z0dDVG6J6V80H62fuxuh+9goLwzDIKiirsUPmnnSzNTOroPSMjwv2ca8e+h/mQ0KIDwlhPoT4uDfYSinDMEjNKWZrah7bqobibz2SR2Z+7d9nTJAXnSP86VRVCdYpwo8gH3fIOwI7vjLbLQ/+TI1dO8O7mgmyjreY1WdyTlpDNA5WvE9puSX0n7kUw4CfplxDqxbNrM1bROQSKSkpYf/+/cTFxeHhoVE4cm733XcfmZmZfPnll+c892w/X3VdPzTjcgcRkXPwDobuvzaPsiLY973Zbrl7MRRlw+Z55uHsDvGDzCRZ2+GNf1i8YZg7JaashJTVZiLs2L7Tz/NtaSbAovtDVF+zOuw8q+hsNhshvu6E+LpzRevgGvflFJU55ped+HNvRgGpOcVkFZSSVVDKyn3ZNR7j7+laY4dMM3HmS4S/R70mzOx2gwPZheZQ/CO5bEs1d4jMKTp9cKmTDVqH+DgG4neK8KdjhB/+nrXPXcAvAvr+1jzy02FnVZLswE/VFY7LnjE3Jek40jxC21/m71ik6flq8xEMA3rHtlBSTEREpB7k5uayZcsW5s6dW6ek2KWixJiISF24eVXPGrNXmjsr7lxoHsf3w55vzIOJ0Kp39bnBbayO/NzKS+DIxqq2yKpEWPHxU06yQWjHqkRYPzMRFhB9WWeuBXi50Ss2kF6xgTVuLyitYO9JybLkqmqzlGNF5BaXs/7gcdYfrBm/t5szrU9OloX60ibUh6hAr4seSF9RaWdvZqGjFXJbah7bj+ZRUFpx2rkuTjbahvnWGIrfoaXvhQ+g9Q2D3veZR2GW+fO440vYtxwytpnH8ukQ3LY6SaZZeSJ1cqKN8ma1UYqISD164IEH+Oijj2q979e//jX/+Mc/6jmi+jNy5EjWrFnDAw88wNChQ+vtedVKKSJyMQwDMnbAroVmy+WRDTXvD2pTnSSL7GXONLNaYbaZ2DsxH+zIRqg8ZTtpF0+zFTKqb1VbZG9zZ88GrKS8kn2ZhSRnFpCcnk9yptmWuT+rkIpadm4EcxOB+GBv2pwyxywmyBs3l9Pfq9KKSnanFZgJsKp5YDuO5lFacfqAVXcXJzq09KNzpJ+5Q2SEP23DfXB3ufhtws+p+Djs+tqsJNu7rOb7GxhfnSRrmdjsk2RaQzQO9f0+7c0sYPBLK3BxsrHm8SEEejeduYYiIvVNrZTnJyMj44wzyvz8/AgNDa3niBq2S9FKqcSYiMillHekanj/Qtj/I9hPap3zDoV2w80kWdxAcK2HhYFhmG2QKSur5oOtNtskT+UdCtF9q+aD9YeWXcH5DK18jUx5pZ2D2UWOyrI9VXPM9mYW1JrQAnB2shEb5EVCqA/xIT5kF5SyNTWP3en5tSbZvN2c6XTSPLDOkf60DvHGxbkBJEJLcmH3N2aSLPk7qCipvi8g2kyQdRhpbkDREBK39UxriMahvt+nV5bs5tWle7imXQizx/e57M8nItKUKTEml5MSY1W0qBWRBqkkF/YsMRNle5ZA6Umf/Lh6Q8JgaH8TtL0OPFtcmuesKDMH4zt2i1wNhZmnnxfczkyEnZgPFhjf7CqHKu0GqceLHYP/95w0x6y2NsgTArxcTxuKHxvkjdNFtmTWi9IC2POtmSTb8y2UF1Xf5xcJHap2t4zq22ySZFpDNA71+T4ZhsE1Ly7nQHYRs+5K5JbuaqUUEbkYSozJ5aTh+yIiDZmHP3S53TwqyuDAj1XVZIsg/4g5C2rHl2BzhpgrzCRZ+xvMKp66Kj4Oh9ZWJ8JS19esCAJwdoOIHjXng3kF1n69ZsTZyUZ0kBfRQV4M7lC9YYJhGKTllTh2yNyXVUALLzezHTLSj8gAzwa78+U5uftA59vMo6zIrCDbvsDcUCIvFVa/ZR4+4dBhhJkki7kCnOqh/VOkgdh8OJcD2UV4uDoxtGMj30xFREREzkmJMRGR+uDiZlaIJQyGG14053qdaLnM2G4mzQ78CIunQHgXaHejmSQL71pdyWUYkHPQHJCfstKsBsvYAZxS+OsZWJ0Ai+4PEYng4l7f33GjZbPZaOnvSUt/T65uE2J1OJePmxd0vNk8ykvMWWTbF5izyQrSYO075uEdYiZtO46E2KuaTIutyJmcGLo/tGM43u5aKouIiDR1+tdeRKS+2WwQ2cM8rp1qzgDbuchMlKWshLQt5rFiJvhHQZvroCjbrAgrSDv9eoGtT6oG62fuhNlYK5rEGq4eZiK2/Q1QUQr7VsCOBWbitjAT1s82D88W5oy8jreYc/JcNJBcmpZKu8H/NpuJsZHdIiyORkREROqDEmMiIlYLjIcrJphHYbbZ1rZrESQvhdxDsO7d6nOdXMydBE9ui/TRzjRyCbm4m3Pv2l4HN80yKxm3L4AdX0FRFmz8yDzc/c1EWseREH9N/WwmIXKZrdqXTWZ+Kf6ergxo24QrRkVERMRBiTERkYbEOwi632MeZUWwbznsX2G2s0X3M2eFuXlZHaU0F86u0Ppa87jhJUj5pSpJ9j8oSIdN/zEPN19od705vD9hiH5GpdFakJQKwA1dWuLm0jw2oBAREWnulBgTEWmo3Lyq29tErObsAnEDzGP48+aMu+1fmomy/COw5VPzcPUy2387jjT/dPexOnKROikpr+TrrWa7+shEtVGKiDR3gwYNIjExkVmzZl2S640bN46cnBzmz59/Sa4nl44SYyIiInJ+nKp2Uo25AoZNN3dD3T7fTJTlplT9fT64eJgVZB1vgbbDwOPM22SLWG35rkzySypo6e9Bn1jt3CsiInKqsrIy3Nya3oxZ1YiLiIjIhXNygqjeMOxvMHEz/OZ7uHIitIiDihLY+RV8fj+80Brm3gVJc6H4uNVRi5zmy01mG+XN3SJwctIGJiIizdm4ceNYsWIFr776KjabDZvNxoEDB9i6dSvDhw/Hx8eHsLAw/u///o+srCzH4/773//SpUsXPD09CQoKYsiQIRQWFjJt2jTef/99FixY4Lje8uXLzxnHlClTaNu2LV5eXsTHx/PXv/6V8vLyGuf873//o3fv3nh4eBAcHMytt97quK+0tJQpU6YQFRWFu7s7CQkJvPuuOb94zpw5BAQE1LjW/PnzsZ20ide0adNITEzkX//6F3FxcXh4mDNlFy9ezFVXXUVAQABBQUHcdNNN7N27t8a1Dh8+zOjRowkMDMTb25tevXqxevVqDhw4gJOTE+vWratx/qxZs4iJicFut5/zdbnUVDEmIiIil8bJO64OmQbpW81Wy23zIXuPubHE7sXmJhLxg8x2y3Y3mrP1RCyUX1LOdzsyALhZbZQiIpeXYUB5kTXP7epVp93bX331VXbv3k3nzp15+umnzYe6utKnTx/uv/9+XnnlFYqLi5kyZQp33nkny5Yt4+jRo4wePZrnn3+eW2+9lfz8fH788UcMw2Dy5Mns2LGDvLw8Zs+eDUBg4Lmrk319fZkzZw4RERFs2bKF3/zmN/j6+vLII48AsHDhQm699VYef/xxPvjgA8rKyli0aJHj8WPGjGHlypW89tprdOvWjf3799dI5NVFcnIyn332GZ9//jnOzs4AFBYWMmnSJLp27UpBQQFPPPEEt956K0lJSTg5OVFQUMDAgQOJjIzkyy+/JDw8nA0bNmC324mNjWXIkCHMnj2bXr16OZ5n9uzZjBs3Dien+q/fUmJMRERELj2bDcK7mMc1j0PmTjNJtn0BZGyH5O/MwzYR4q42k2Ttb9Iuq2KJb7alU1ZhJyHUh44t1fIrInJZlRfBdIs+hPjLEXDzPudp/v7+uLm54eXlRXh4OADPPvss3bt3Z/r06Y7z3nvvPaKioti9ezcFBQVUVFRw2223ERMTA0CXLl0c53p6elJaWuq4Xl1MnTrV8ffY2FgmT57MvHnzHImxv/3tb9x999089dRTjvO6desGwO7du/nkk09YsmQJQ4YMASA+Pr7Oz31CWVkZH3zwASEh1bs1jxo1qsY57733HiEhIWzfvp3OnTszd+5cMjMzWbt2rSMBmJCQ4Dj//vvv54EHHuDll1/G3d2dDRs2sGXLFhYsWHDe8V0KaqUUERGRy8tmg9AOMOhR+MNKeHAtXDsVwruCUWnuvvrVn+CldjDnJlj9T8g7anXU0oyc2I1yZLeIGi0kIiIiJ2zatInvv/8eHx8fx9G+fXsA9u7dS7du3Rg8eDBdunThjjvu4J133uH48YsbH/Hxxx9z5ZVXEh4ejo+PD1OnTiUlJcVxf1JSEoMHD671sUlJSTg7OzNw4MCLiiEmJqZGUgxgz549jB49mvj4ePz8/IiNjQVwxJaUlET37t3PWBV3yy234OzszBdffAGYbZ3XXHON4zr1TRVjIiIiUr9C2kLIwzDgYTi2r3p3yyMb4MCP5vH1IxDV16wk63gz+LeyOmppojLzS/k52WwrURuliEg9cPUyK7eseu4LVFBQwIgRI3juuedOu69ly5Y4OzuzZMkSfvnlF7799ltef/11Hn/8cVavXk1cXNx5P9/KlSu55557eOqppxg2bBj+/v7MmzePl156yXGOp6fnGR9/tvsAnJycMAyjxm2nzi8D8PY+vcJuxIgRxMTE8M477xAREYHdbqdz586UlZXV6bnd3NwYM2YMs2fP5rbbbmPu3Lm8+uqrZ33M5aSKMREREbFOYDxcNRF++z38cTNc9zdo1Qcw4NAq+OYxeDURSvIsDrT5ePPNN4mNjcXDw4O+ffuyZs2aM55bXl7O008/TevWrfHw8KBbt24sXrz4oq5Z3xZuPoLdgMSoAGKCzt1eIyIiF8lmM9sZrTjOoyrYzc2NyspKx9c9evRg27ZtxMbGkpCQUOM4kTyy2WxceeWVPPXUU2zcuBE3NzdHVdSp1zuXX375hZiYGB5//HF69epFmzZtOHjwYI1zunbtytKlS2t9fJcuXbDb7axYsaLW+0NCQsjPz6ewsNBxW1JS0jnjys7OZteuXUydOpXBgwfToUOH0yrjunbtSlJSEseOHTvjde6//36+++47/v73vztaUK2ixJiIiIg0DC1i4IoJcP8S+NN2uP45iL7CHNTvoblP9eHjjz9m0qRJPPnkk2zYsIFu3boxbNgwMjIyaj1/6tSpvP3227z++uts376dBx54gFtvvZWNGzde8DXrm5e7C3HB3oxUtZiIiJwkNjbWsYtiVlYWDz74IMeOHWP06NGsXbuWvXv38s033zB+/HgqKytZvXo106dPZ926daSkpPD555+TmZlJhw4dHNfbvHkzu3btIisrq9bqrJO1adOGlJQU5s2bx969e3nttdccSbYTnnzySf7zn//w5JNPsmPHDrZs2eKoaIuNjWXs2LHce++9zJ8/n/3797N8+XI++eQTAPr27YuXlxd/+ctf2Lt3L3PnzmXOnDnnfF1atGhBUFAQ//znP0lOTmbZsmVMmjSpxjmjR48mPDycW265hZ9//pl9+/bx2WefsXLlSsc5HTp0oF+/fkyZMoXRo0efs8rssjIuwBtvvGHExMQY7u7uRp8+fYzVq1ef9fxPPvnEaNeuneHu7m507tzZWLhwYY37x44dawA1jmHDhtU5ntzcXAMwcnNzL+TbERERkYasovyyXVpriJr69OljPPjgg46vKysrjYiICGPGjBm1nt+yZUvjjTfeqHHbbbfdZtxzzz0XfM3aXO73yW63G2UVlZfl2iIizV1xcbGxfft2o7i42OpQzsuuXbuMfv36GZ6engZg7N+/39i9e7dx6623GgEBAYanp6fRvn17Y+LEiYbdbje2b99uDBs2zAgJCTHc3d2Ntm3bGq+//rrjehkZGcbQoUMNHx8fAzC+//77c8bw8MMPG0FBQYaPj49x1113Ga+88orh7+9f45zPPvvMSExMNNzc3Izg4GDjtttuc9xXXFxs/OlPfzJatmxpuLm5GQkJCcZ7773nuP+LL74wEhISDE9PT+Omm24y/vnPfxonp4mefPJJo1u3bqfFtWTJEqNDhw6Gu7u70bVrV2P58uUGYHzxxReOcw4cOGCMGjXK8PPzM7y8vIxevXqdljt69913DcBYs2bNOV+LMznbz1dd1w82wzilqfQcPv74Y8aMGcM//vEP+vbty6xZs/j000/ZtWsXoaGn7yT1yy+/MGDAAGbMmMFNN93E3Llzee6559iwYQOdO3cGYNy4caSnpzu2LQVwd3enRYsWdYopLy8Pf39/cnNz8fPTJ8oiIiJSN1pDVCsrK8PLy4v//ve/3HLLLY7bx44dS05OTq07RQUFBfH8889z3333OW779a9/zU8//cSBAwcu6JoApaWllJaWOr7Oy8sjKipK75OISCNUUlLC/v37iYuLw8PDw+pwpAF55pln+PTTT9m8efMFX+NsP191Xeeddyvlyy+/zG9+8xvGjx9Px44d+cc//oGXlxfvvfderee/+uqrXH/99Tz88MN06NCBZ555hh49evDGG2/UOM/d3Z3w8HDHUdekmIiIiIhcvKysLCorKwkLC6txe1hYGGlpabU+ZtiwYbz88svs2bMHu93OkiVL+Pzzzzl69OgFXxNgxowZ+Pv7O46oqKiL/O5ERESkoSgoKGDr1q288cYbPPTQQ1aHc36JsbKyMtavX8+QIUOqL+DkxJAhQ2r0ip5s5cqVNc4HcxF16vnLly8nNDSUdu3a8fvf/57s7OwzxlFaWkpeXl6NQ0RERETq16uvvkqbNm1o3749bm5uTJgwgfHjx+PkdHFjbB977DFyc3Mdx6FDhy5RxCIiIg3D9OnT8fHxqfUYPny41eFdVhMmTKBnz54MGjSIe++91+pwcDmfk8/2qd/OnTtrfUxaWto5PyW8/vrrue2224iLi2Pv3r385S9/Yfjw4axcuRJnZ+fTrjljxgyeeuqp8wldRERERM4iODgYZ2dn0tPTa9yenp5OeHh4rY8JCQlh/vz5lJSUkJ2dTUREBI8++ijx8fEXfE0wOwnc3d0v8jsSERFpuB544AHuvPPOWu+zdBB9PZgzZ06dBv3Xl/NKjF0ud999t+PvXbp0oWvXrrRu3Zrly5czePDg085/7LHHaux6cGLuhIiIiIhcGDc3N3r27MnSpUsd88DsdjtLly5lwoQJZ32sh4cHkZGRlJeX89lnnzkW+hdzTRERkaYsMDCQwMBAq8MQzjMxdiGf+oWHh5/3p4Tx8fEEBweTnJxca2JMnyKKiIiIXHqTJk1i7Nix9OrViz59+jBr1iwKCwsZP348AGPGjCEyMpIZM2YAsHr1alJTU0lMTCQ1NZVp06Zht9t55JFH6nxNERERESudV2LsQj7169+/P0uXLmXixImO25YsWUL//v3P+DyHDx8mOzubli1bnk94IiIiInIR7rrrLjIzM3niiSdIS0sjMTGRxYsXO8ZipKSk1JgfVlJSwtSpU9m3bx8+Pj7ccMMNfPjhhwQEBNT5miIi0jwYhmF1CNIE2e32i76GzTjPn86PP/6YsWPH8vbbbzs+9fvkk0/YuXMnYWFhp32S+MsvvzBw4EBmzpzJjTfeyLx585g+fTobNmygc+fOFBQU8NRTTzFq1CjCw8PZu3cvjzzyCPn5+WzZsqVOlWHaal1EREQuhNYQjYPeJxGRxquyspI9e/bg5eVFSEgINpvN6pCkCTAMg7KyMjIzM6msrKRNmzanbf5T1/XDec8YO99PEq+44grmzp3L1KlT+ctf/kKbNm2YP38+nTt3BsDZ2ZnNmzfz/vvvk5OTQ0REBNdddx3PPPOM2iVFREREREREGjFnZ2datWrF4cOHOXDggNXhSBPj5eVFdHT0Re2Ifd4VYw2RPkUUERGRC6E1ROOg90lEpPGrrKykvLzc6jCkCXF2dsbFxeWMVYiXrWJMREREREREROR8ODs74+zsbHUYIqe58FozERERERERERGRRkyJMRERERERERERaZaUGBMRERERERERkWapScwYO7F/QF5ensWRiIiISGNyYu3QBPYiatK01hMREZHzVdd1XpNIjOXn5wMQFRVlcSQiIiLSGOXn5+Pv7291GHIGWuuJiIjIhTrXOs9mNIGPSO12O0eOHMHX1/eM23RejLy8PKKiojh06JC2CLeI3gNr6fW3ll5/a+n1t9blfv0NwyA/P5+IiAicnDRhoqHSWq9p0+tvLb3+1tLrby29/tZqKOu8JlEx5uTkRKtWrS778/j5+ek/FovpPbCWXn9r6fW3ll5/a13O11+VYg2f1nrNg15/a+n1t5Zef2vp9beW1es8fTQqIiIiIiIiIiLNkhJjIiIiIiIiIiLSLCkxVgfu7u48+eSTuLu7Wx1Ks6X3wFp6/a2l199aev2tpddf6oN+zqyl199aev2tpdffWnr9rdVQXv8mMXxfRERERERERETkfKliTEREREREREREmiUlxkREREREREREpFlSYkxERERERERERJolJcZERERERERERKRZUmKsDt58801iY2Px8PCgb9++rFmzxuqQmo0ffviBESNGEBERgc1mY/78+VaH1GzMmDGD3r174+vrS2hoKLfccgu7du2yOqxm46233qJr1674+fnh5+dH//79+frrr60Oq9maOXMmNpuNiRMnWh1KszFt2jRsNluNo3379laHJU2Q1nnW0TrPOlrnWUvrvIZF67z619DWeUqMncPHH3/MpEmTePLJJ9mwYQPdunVj2LBhZGRkWB1as1BYWEi3bt148803rQ6l2VmxYgUPPvggq1atYsmSJZSXl3PddddRWFhodWjNQqtWrZg5cybr169n3bp1XHvttYwcOZJt27ZZHVqzs3btWt5++226du1qdSjNTqdOnTh69Kjj+Omnn6wOSZoYrfOspXWedbTOs5bWeQ2H1nnWaUjrPJthGIZlz94I9O3bl969e/PGG28AYLfbiYqK4qGHHuLRRx+1OLrmxWaz8cUXX3DLLbdYHUqzlJmZSWhoKCtWrGDAgAFWh9MsBQYG8sILL3DfffdZHUqzUVBQQI8ePfj73//Os88+S2JiIrNmzbI6rGZh2rRpzJ8/n6SkJKtDkSZM67yGQ+s8a2mdZz2t8+qf1nnWaWjrPFWMnUVZWRnr169nyJAhjtucnJwYMmQIK1eutDAykfqXm5sLmP9oS/2qrKxk3rx5FBYW0r9/f6vDaVYefPBBbrzxxhr/Dkj92bNnDxEREcTHx3PPPfeQkpJidUjShGidJ1JN6zzraJ1nHa3zrNWQ1nkulj1zI5CVlUVlZSVhYWE1bg8LC2Pnzp0WRSVS/+x2OxMnTuTKK6+kc+fOVofTbGzZsoX+/ftTUlKCj48PX3zxBR07drQ6rGZj3rx5bNiwgbVr11odSrPUt29f5syZQ7t27Th69ChPPfUUV199NVu3bsXX19fq8KQJ0DpPxKR1njW0zrOW1nnWamjrPCXGROScHnzwQbZu3ar5PvWsXbt2JCUlkZuby3//+1/Gjh3LihUrtGiqB4cOHeKPf/wjS5YswcPDw+pwmqXhw4c7/t61a1f69u1LTEwMn3zyidpMREQuIa3zrKF1nnW0zrNeQ1vnKTF2FsHBwTg7O5Oenl7j9vT0dMLDwy2KSqR+TZgwga+++ooffviBVq1aWR1Os+Lm5kZCQgIAPXv2ZO3atbz66qu8/fbbFkfW9K1fv56MjAx69OjhuK2yspIffviBN954g9LSUpydnS2MsPkJCAigbdu2JCcnWx2KNBFa54lonWclrfOso3Vew2P1Ok8zxs7Czc2Nnj17snTpUsdtdrudpUuXqv9bmjzDMJgwYQJffPEFy5YtIy4uzuqQmj273U5paanVYTQLgwcPZsuWLSQlJTmOXr16cc8995CUlKTFkgUKCgrYu3cvLVu2tDoUaSK0zpPmTOu8hkfrvPqjdV7DY/U6TxVj5zBp0iTGjh1Lr1696NOnD7NmzaKwsJDx48dbHVqzUFBQUCNrvH//fpKSkggMDCQ6OtrCyJq+Bx98kLlz57JgwQJ8fX1JS0sDwN/fH09PT4uja/oee+wxhg8fTnR0NPn5+cydO5fly5fzzTffWB1as+Dr63vanBVvb2+CgoI0f6WeTJ48mREjRhATE8ORI0d48skncXZ2ZvTo0VaHJk2I1nnW0jrPOlrnWUvrPGtpnWe9hrbOU2LsHO666y4yMzN54oknSEtLIzExkcWLF582qFUuj3Xr1nHNNdc4vp40aRIAY8eOZc6cORZF1Ty89dZbAAwaNKjG7bNnz2bcuHH1H1Azk5GRwZgxYzh69Cj+/v507dqVb775hqFDh1odmki9OHz4MKNHjyY7O5uQkBCuuuoqVq1aRUhIiNWhSROidZ61tM6zjtZ51tI6T5q7hrbOsxmGYVjyzCIiIiIiIiIiIhbSjDEREREREREREWmWlBgTEREREREREZFmSYkxERERERERERFplpQYExERERERERGRZkmJMRERERERERERaZaUGBMRERERERERkWZJiTEREREREREREWmWlBgTEREREREREZFmSYkxERERERERERFplpQYExERERERERGRZkmJMRERERERERERaZaUGBMRERERERERkWbp/wMo2QIS0coAXAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt # Visualization\n", + "\n", + "# Plot loss and accuracy in subplots\n", + "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 5))\n", + "ax1.set_title('Loss')\n", + "ax2.set_title('Accuracy')\n", + "for dataset in ('train', 'test'):\n", + " ax1.plot(metrics_history[f'{dataset}_loss'], label=f'{dataset}_loss')\n", + " ax2.plot(metrics_history[f'{dataset}_accuracy'], label=f'{dataset}_accuracy')\n", + "ax1.legend()\n", + "ax2.legend()\n", + "plt.show()" ] }, { @@ -387,14 +504,14 @@ "id": "25", "metadata": {}, "source": [ - "## 7. Perform inference on the test set\n", + "## 10. Perform inference on the test set\n", "\n", "Create a `jit`-compiled model inference function (with `nnx.jit`) - `pred_step` - to generate predictions on the test set using the learned model parameters. This will enable you to visualize test images alongside their predicted labels for a qualitative assessment of model performance." ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "id": "26", "metadata": {}, "outputs": [], @@ -417,7 +534,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "id": "27", "metadata": { "outputId": "1db5a01c-9d70-4f7d-8c0d-0a3ad8252d3e" @@ -425,7 +542,7 @@ "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7QAAAPGCAYAAADTLdZkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAACY0UlEQVR4nOzde5yN9f7//9cawxyI7RhTQs4ZUeQwxUhSEckWKadShkK2xFZyTCVF7ewyysepnEKkiGwjOaSQDsoelWmLCjkzZpi5fn/0Nb+m63XVWjNrzTXvaz3ut5vbLc95977es7zfs9ZrrpnX8lmWZQkAAAAAAIaJcHsBAAAAAADkBQUtAAAAAMBIFLQAAAAAACNR0AIAAAAAjERBCwAAAAAwEgUtAAAAAMBIFLQAAAAAACNR0AIAAAAAjERBCwAAAAAwUlgVtLNnzxafzydpaWkB/X+tWrWS+Pj4oK6latWq0qdPn6DOCfwZ9j/CHWcA4Yz9j3DHGfCusCpovejTTz+VgQMHSr169aR48eJyxRVXSNeuXSU1NdXtpQEFIiMjQ0aMGCFxcXESExMjTZs2lQ8++MDtZQGumDhxovh8vqC/+AIKq71798rdd98tl19+ucTGxkqdOnVk/PjxcvbsWbeXBoRcnz59xOfzOf45cOCA20ssEJFuLwD5M2nSJNm8ebPcddddcvXVV8vPP/8s06ZNk2uvvVY+/vhjXtTA8/r06SNLliyRIUOGSM2aNWX27NnSrl07SUlJkRtuuMHt5QEF5scff5Snn35aihcv7vZSgAKxf/9+adKkiZQqVUoGDhwoZcqUka1bt8qYMWNkx44dsmLFCreXCIRUUlKStGnTJldmWZb0799fqlatKpdddplLKytYFLSGGzp0qMyfP1+KFSuWk3Xr1k3q168vzz77rLzxxhsurg4IrU8++UQWLlwokydPlmHDhomISK9evSQ+Pl6GDx8uW7ZscXmFQMEZNmyYNGvWTLKysuTIkSNuLwcIuXnz5snx48dl06ZNUq9ePRER6devn2RnZ8vcuXPl2LFjUrp0aZdXCYRO8+bNpXnz5rmyTZs2ydmzZ+Xee+91aVUFL6x/5HjFihXSvn17iYuLk6ioKKlevbpMmDBBsrKy1PE7duyQhIQEiYmJkWrVqsn06dNtYzIyMmTMmDFSo0YNiYqKksqVK8vw4cMlIyMjJJ9DQkJCrmJWRKRmzZpSr149+eabb0JyTXiDF/b/kiVLpEiRItKvX7+cLDo6Wvr27Stbt26V/fv3h+S68AYvnIGLNm7cKEuWLJEXX3wxpNeBd3hh/588eVJERC699NJceaVKlSQiIsL2+gj4PS+cAc38+fPF5/PJPffcU2DXdFtY36GdPXu2lChRQoYOHSolSpSQ9evXy+jRo+XkyZMyefLkXGOPHTsm7dq1k65du0r37t1l8eLFMmDAAClWrJjcf//9IiKSnZ0tHTt2lE2bNkm/fv2kbt268uWXX8rUqVMlNTVVli9f7riW7OxsOXr0qF/rLlWqlBQtWtTx45ZlyS+//JLz3UpA44X9/9lnn0mtWrWkZMmSucY0adJERER27dollStX9vchQZjxwhkQEcnKypJBgwbJAw88IPXr1w/8gUBY8sL+b9WqlUyaNEn69u0r48aNk7Jly8qWLVvk1VdflcGDB/Pj9/hTXjgDf3T+/HlZvHixJCQkSNWqVf2azxOsMDJr1ixLRKx9+/ZZlmVZZ8+etY1JSkqyYmNjrXPnzuVkiYmJlohYL7zwQk6WkZFhNWzY0KpQoYKVmZlpWZZlzZs3z4qIiLA++uijXHNOnz7dEhFr8+bNOVmVKlWs3r175/x93759loj49SclJeVPP8958+ZZImLNnDnT34cGYcCL+79evXpW69atbZ/H7t27LRGxpk+fHtBjBG/z4hmwLMuaNm2aVapUKevQoUM5661Xr16eHiN4l1f3/4QJE6yYmJhcY5544om8PkzwMK+egd9buXKlJSLWK6+8EshDY7ywvkMbExOT89+nTp2SjIwMadGihSQnJ8uePXukQYMGOR+PjIyUpKSknL8XK1ZMkpKSZMCAAbJjxw5p1qyZvPXWW1K3bl2pU6dOrt9fat26tYiIpKSkSEJCgrqWihUr+t2Z9ffr+qM9e/bIww8/LM2bN5fevXv7NR/Ckxf2f3p6ukRFRdnGREdH53wccOKFM/Drr7/K6NGj5cknn5Ty5cv794kD4o39L/Lb25+0bNlS/v73v0vZsmXlvffek6effloqVqwoAwcO9GtOhCevnIHfmz9/vhQtWlS6du3q11xeEdYF7e7du2XUqFGyfv36nN/DuOjEiRO5/h4XF2f70ZVatWqJiEhaWpo0a9ZM9u7dK998843ji4pDhw45riU6OtrWpSxQP//8s7Rv315KlSqV87uFgBMv7P+YmBj191LOnTuX83HAiRfOwKhRo6RMmTIyaNCggP9fhDcv7P+FCxdKv379JDU1VS6//HIREencubNkZ2fLiBEjpHv37lK2bNmA50V48MIZ+L3Tp0/LihUr5JZbbgm7fR+2Be3x48clMTFRSpYsKePHj5fq1atLdHS07Ny5U0aMGCHZ2dkBz5mdnS3169eXKVOmqB//s9/ly8rKksOHD/t1nTJlytgaHZw4cUJuu+02OX78uHz00UcSFxfn/8IRdryy/ytVqqS+x9pPP/0kIsI5gCMvnIG9e/fKjBkz5MUXX5SDBw/mfPzcuXNy/vx5SUtLk5IlS0qZMmUC+0TgeV7Y/yIir7zyilxzzTU5xexFHTt2lNmzZ8tnn32W7yIB3uSVM/B7y5cvD7vuxheFbUG7YcMG+fXXX2XZsmXSsmXLnHzfvn3q+IMHD8qZM2dyfXcmNTVVRCTnl66rV68un3/+udx0003i8/kCWs/+/fulWrVqfo1NSUmRVq1a5fz93Llz0qFDB0lNTZV169bJVVddFdC1EX68sv8bNmwoKSkpcvLkyVyNobZt25bzcUDjhTNw4MAByc7OlsGDB8vgwYNt46pVqyaPPPIInY9h44X9LyLyyy+/qG/Lc/78eRERuXDhQkDrQPjwyhn4vTfffFNKlCghHTt2DOjaXhC2Be3FH8e1LCsny8zMlFdeeUUdf+HCBUlOTpahQ4fmjE1OTpby5ctLo0aNRESka9eusmrVKnnttddyvY2IyG+/y5edne3YcS+vPzuflZUl3bp1k61bt8qKFSts70UFaLyy/7t06SLPP/+8zJgxI+d9aDMyMmTWrFnStGlTOhzDkRfOQHx8vLz99tu2j48aNUpOnTolL730klSvXt2vORFevLD/RX77kc+1a9dKampqzo9/iogsWLBAIiIi5Oqrr/ZrToQfr5yBiw4fPizr1q2T7t27S2xsrF/zeEnYFrQJCQlSunRp6d27twwePFh8Pp/Mmzcv18b+vbi4OJk0aZKkpaVJrVq1ZNGiRbJr1y6ZMWNGTuvsnj17yuLFi6V///6SkpIi119/vWRlZcmePXtk8eLFsmbNGmncuLE6f15/dv7RRx+Vd955Rzp06CBHjx6VN954I9fHe/ToEfCc8D6v7P+mTZvKXXfdJSNHjpRDhw5JjRo1ZM6cOZKWliYzZ84MeD6EDy+cgXLlykmnTp1s+cU7strHABFv7H8Rkccee0xWr14tLVq0kIEDB0rZsmXl3XffldWrV8sDDzzAr53AkVfOwEWLFi2SCxcuhOWPG4tIeL9tz+bNm61mzZpZMTExVlxcnDV8+HBrzZo1tpbYF98CYfv27Vbz5s2t6Ohoq0qVKta0adNs18jMzLQmTZpk1atXz4qKirJKly5tNWrUyBo3bpx14sSJnHF/bNedVxdbiTv9AS7y4v63LMtKT0+3hg0bZlWsWNGKioqyrrvuOuv9998PytzwFq+egT/ibXug8er+37Ztm3XbbbdZFStWtIoWLWrVqlXLmjhxonX+/PmgzA/v8OoZsCzLatasmVWhQgXrwoULQZvTJD7LcvhWBAAAAAAAhViE2wsAAAAAACAvKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGCnS34E+ny+U6wD+lNtvl8z+h5vc3v8inAG4y+0zwP6Hm9ze/yKcAbjrr84Ad2gBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaKdHsByL9KlSqpeZkyZWzZhQsX1LH//e9/g7omFF7XXnutmvft21fNBwwYoOYrVqywZWvXrs37wv6fr7/+Ws0//PDDfM8NAAAAb+EOLQAAAADASBS0AAAAAAAjUdACAAAAAIxEQQsAAAAAMBIFLQAAAADASD7Lsiy/Bvp8oV4L/kKNGjXUPCUlRc217sfnz59Xx7766qtqPnToUD9XF1p+btOQMXX/N2zY0JatWrVKHXvppZeGeDX+OXbsmJpv3LhRzadMmaLmP/74oy1LS0vL87rc5Pb+FzH3DMAb3D4D7H+4ye39L8IZgLv+6gxwhxYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJplD50LJlS1v21ltvqWOdHuZZs2b5Na+ISHx8vJqXKFEioGtqnJpFbd682Za1adPG73mDxe2GCIV9/2vNn0REli1bZsuqVKkS4tXkj9NjHege+Prrr23Z/Pnz1bHPP/+8mjudi4Lm9v4XKTxnwOnr4Pr169U8OTnZlj355JNBXZPbevTooeZ33XWXLbv//vvVsb/++mtQ1xRsbp+BwrL/w5lT48J77rlHzZ2eFzUvv/yymm/fvt3vOULJ7f0vwhmAu2gKBQAAAADwJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJLoc++Fvf/ubmu/YscOWVa1aVR0bjA51Bw8eVPOhQ4f6PceYMWPUvG7dumq+du1aW9auXTu/rxcsbnf4K+z7//PPP1dzp46whVmwuhwHwqnD5ZAhQ0J2zUC4vf9FCs8ZmDJlipr/4x//UPMvvvjClt1xxx3q2LS0tDyvy027d+9W86uuusqWLVmyRB2rdUQuTNw+A4Vl/3tNkSJFbNnw4cPVsU6vdZz+bcqUKeP3Ov7zn/+o+c033+z3HKHk9v4XKTxnoE6dOmr+4osvqvlll11my5y6VzvN4fQaCwWHLscAAAAAAE+ioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaKdHsBhUmTJk3U/KmnnlLzKlWq5Puas2bNsmXff/+932NFRH7++We/rzdhwgS/x4qIfPfddwGNR/jYs2ePLXPqHpuRkaHm3bt3t2UtWrRQxzp1G09ISHBYof8eeughNde6Oj766KPq2AsXLuR7HchN+ze//PLL8z1HVFRUHlfkLqcu+rGxsX7PcdNNNwVpNYD/GjRooOZjx461ZU7PI3PmzFHzcePGqfn+/ftt2dy5c9WxrVu3VvNAVKxYUc0DeY2Gv3bppZeq+S233OL3HE7vANGjRw81T01NVfNNmzb5fU0nq1atsmXp6enq2M6dO6v5ggUL8r0Opy7/P/zwQ77nLgjcoQUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEbyWZZl+TVQaY7iNVpzAhGRJ5980u85Nm/erOZa8xsRkQMHDvg9dzD88ssval6uXDk11xpijRkzJqhr8oef2zRkCsv+b9++vZq/+eaban7JJZfk+5qHDx9W8+uvv96WhbKJWJkyZdT8xhtvVPMZM2bYMqfGUoGoXr26mjs1VAgGt/e/iDtnoGXLlrbsww8/DGgO7WtYIF/TC5OJEyeq+eOPP+73HMeOHVNzp/NVWLh9BgrLc0Bh16xZMzWfPXu2mmtfT/v376+OdWqMmZ2d7d/iROSyyy5T89WrV6v5fffdZ8ucXgN9/vnnah6Mrzdu73+RwnMGnJr6Oe0Pp9ffyO3UqVNq/sknn9iyNm3ahHo5Nn91BrhDCwAAAAAwEgUtAAAAAMBIFLQAAAAAACNR0AIAAAAAjERBCwAAAAAwUqTbCyhMdu/ereZvvfWWmn/11Ve2TOuo6ZYHHnjAlpUsWVId69Q9bNGiRUFdE/LniiuuUPNgdDN2smDBAjUPZUdjzdGjR9V86dKlal6zZk1b5tQlNhArV65U8w4dOqh5KLsfe53WqTocNGjQQM0feuihfM/9ww8/5HsOwMmjjz6q5rVr11bzO+64w5a98847QV3T7505c0bN4+Li1PzTTz+1ZaNHj1bHTpkyJe8Lg98yMjLU/P7771fz8ePH27JbbrlFHXvy5Ek179Wrl5pXrlxZzUOlUqVKau7U6btEiRJ+z+30OvKzzz7zew43cYcWAAAAAGAkCloAAAAAgJEoaAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkn+XU3vaPA32+UK8FQbZ+/Xpb1rJlS3Xsf/7zHzVv3769Lbtw4UL+FpYHfm7TkCks+z89PV3NixUrFrJrpqamqnndunVDds1giIqKsmUdO3ZUxy5cuDDf19O6YYqINGvWLN9zu73/Rdw5A8ePH7dlpUqVCmgOrfP8k08+mdclFYgmTZqo+bZt2/I9d4sWLdR806ZN+Z47lNw+A4XlOaAwqVq1qi1z6n7/2muvqfmAAQNsWbD+rbV3BXj55ZfVsbfffruaa53W//GPf6hjz507F8DqAuP2/hfhDBQGtWrVUnOn12PLli2zZRER+r3MrKwsNe/bt68tmzNnjtMSQ+avzgB3aAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkCloAAAAAgJEoaAEAAAAARop0ewHIv6ZNm6r5VVdd5fccTh0I3ehoDGda516R0HZArFKlipr36NHDlr3xxhshW0egMjIybJlTN+8tW7aoeUJCgt/Xi46O9nsschs3bpyalyhRwu85nLqrTp8+PU9rAvDnKlasaMucOuF++OGHaq49d0VG6i9NtY7IIiKtW7dW81tvvdWWffvtt+rYLl26qPnbb7+t5oAb9u7dq+bPPvusmmsdjZ1eLz722GNq7kZH47zgDi0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADASTaEMEh8fr+bvvfeemv/tb3+zZRs3blTHrl27Ns/rgrc5NaK67LLLCngl+Xf06FE1P378eMEuBLk4NR4rUqSI33PExsaq+eWXX27LDhw44Pe8AHQNGzb0e+yRI0fUvH///rbs4YcfVsfWq1dPzY8dO6bmkyZNsmUvv/yyOvbXX39Vc6AwadWqlZrfeeedfs8xZcoUNZ86dWpellRocIcWAAAAAGAkCloAAAAAgJEoaAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkz3c51rriderUSR3bsWNHNW/cuLHf14uI0L9HkJ2dreaffvqpX5mISPfu3dW8bNmyaq51bh07dqw69uTJk2qOwmXTpk1qfsMNNxTwSkR8Pl+BXzNUBg4cqOb79u2zZU6f99VXX63mAwYMUPNXX33Vz9V53/PPP6/m2tfk0qVLq2MrVaqk5gsWLLBl3377bQCrK3ilSpUK2dzjx49X81tvvVXNMzMzQ7YWmM3ptYfm3XffVfPISPvL0M8++0wde99996n5woUL1TwjI8PP1QGFywMPPKDmr732WkDzaO/sMHHixDytqbDjDi0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEjGdTnu0qWLmj/00ENqnpiYaMssywromoGMd+pm7DSH1kE5kK7Kf3ZN7THZuHFjQHOjcNE6toqIXH/99fme26m79k8//aTmM2fOzPc1C4srr7xSzbVzG8qvH+Hqq6++UvOEhARbtnz5cnVs7dq11bxatWp+ZeHixhtvVPPp06er+f333x/K5cAAbdu2VfMRI0b4PYdTt+w77rjDlr3//vt+zwuY7vLLL7dljzzySFDmTkpKsmXHjh0LytyFDXdoAQAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkQp1U6g777zTls2dO1cdW6xYMTU/fPiwLXNq0jJr1iw1P3funJovXLjQljn9svX48ePV/MEHH1TzYDh48GDI5ob33HXXXWq+f//+Al5JwRs6dGi+53B6nNatW5fvucPVnj17bNndd9+tjm3Tpo2aT548OahrMt3p06fV3KkpFMJH37591XzGjBlq/u2339qyQ4cOqWMbNWqk5kWLFvVzdYA3LV261JbFx8cHNIfT12+nJopexB1aAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRCkWX4y5duqi51tHYqZuxU4fiUHYR1owePVrNtY7NoXbvvffasq1bt6pjMzMzQ70cwHU1atRQ8+rVq+d77uPHj6u51gkUebdr1y41/+KLL9R82rRptuyFF15Qx6ampqp5cnKymrdo0cKWDRs2TB0biFatWqm50/Ofk5deesmWjRgxQh2bkZER0Nwww6WXXmrLnnvuOXVsu3bt1Nyp+/H8+fNt2RVXXKGOdXqNpp3PTz/9VB37888/qzlgghtuuEHNGzRo4PccW7ZsUfMBAwbkaU1ewh1aAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRfJZlWX4N9PlCtoj169erecuWLW2ZU6e8gQMHqnkwOjdedtllav7EE0/YsqSkJHWs08OsdfN7+umn1bH33Xefmt9xxx1+X/Mf//iHOvbll19W88LCz20aMqHc/4EoUaKEmn/yySdqXrt2bb/nfuONN9S8d+/efs9RmGgdjd999111bM2aNfN9Pa3jp4hIz5498z232/tfpPCcgXDw008/qXnFihXV/MiRI2quPTc4dcks7Nw+A4V9/0dG6m9a8euvv9oyp8+ldevWar59+/a8L+z/6dq1q5ovXLjQljm9K8SKFSvyvQ5Tub3/RQr/GSgsGjdurOabN29Wc617/YIFC9SxDz30kJo7vcuCl/zVGeAOLQAAAADASBS0AAAAAAAjUdACAAAAAIxEQQsAAAAAMBIFLQAAAADASHpbvBC54YYb1DwxMVHN//vf/9qyBx98MN/rqFq1qpq3atVKzR9//HE1r169ui3LzMxUxz7//PNqrnXtc+oouHLlSjXXuhiKiPztb3+zZZ07d1bHzpkzR81Pnjyp5nDH6dOn1fz8+fP5nrtt27ZqPnfuXDUfNGiQLTtx4kS+1+EkOjpazatUqaLmb7/9ti0LRjfjH3/8Uc1feumlfM8N5IXTuTO1ozGcFS1aVM03btyo5to7PTh9rd+1a1ee1/VXypYt6/dYp67dQGETEWG/L+j0mknrZiwism3bNlsWzt2M84o7tAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEgF2hTqiSeeUHPLstR84cKFfs9do0YNNb/pppts2dNPP62OLVWqlN/XExFZs2aNLRs9erQ61qnRUzC0a9dOzZcvX27LWrRooY7997//reY9e/bM87pQcLTmYiIi8fHxfs9RoUIFNb/33nvV/PLLL7dlH3/8sTr2nXfeUfOOHTvaMp/P5/f1RETuueceNQ+V+vXrqzkN1AAEU7ly5WzZhAkT1LFNmzZV84SEBFsWyuZPUVFRau70WkJrpJmamhrUNQGhMmvWLFtWt25ddazTa4Rhw4bZMpo/BY47tAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEgUtAAAAAAAIxVol+O2bduquVOX48TERFu2efNmdaxTN9cSJUrYsnPnzqlj//e//6m5UxdVrXPxhQsX1LGhtG3bNjXfunWrLevQoYM6VuuEKCJy22232bLVq1cHsDoUhPHjx6v5qVOnbNmzzz4blGtq51PLREQeeeQRNY+OjrZlERH699mys7MDWF1gli1bpuZ9+/a1ZdpjCuSV1hlf626L8HPkyBFbFhsbq449evSommtfYyMjA3vp17BhQzWvXLmyLZsyZYrfY0X0567Dhw/7vzigADz88MNq3qtXL7/n+Ne//qXmmzZtytOakBt3aAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkCloAAAAAgJEoaAEAAAAARirQLsezZs1S8z59+qi51jH166+/VsfOnj1bzT/66CNb9uOPP6pjP/74YzU3VefOnW3ZnDlz1LH33nuvmmvdDelyXPg4ddeeOnWqLdM6f4uIjBgxQs2LFi2a94X9P1qnTSdOXc8DpXXK/OCDD9SxgwcPVvOTJ08GZS2Ak0qVKtmyQLvQLl++PEirQWGnfU0XcX6nh/Xr14dsLVrn+Q8//FAde/vtt6v57t27g7omID9iYmLU3Kl7t2bt2rVqPnny5DytCf7hDi0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADCSz/KzA4vP58v3xaKiotS8evXqfs/h1NCJ5i3+KV++fED5d999Z8syMjKCuiZ/BKtRUF4FY/8Xdj169FDzypUrq/lTTz0VknVEROjfZ0tNTVVzpyYpn332mS3btm1b3hfmIrf3v0h4nAE3vPrqq7asf//+Ac3h1BDISw133D4DhX3/V6xYUc1vuummfM/9ww8/qPmePXts2ZEjR/J9Pdi5vf9FCv8ZCIaJEyeq+eOPP67m3377rS27+uqr1bHp6el5Xxj+8gxwhxYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYKQC7XIM5JXbHf7Y/3CT2/tfhDMQKnQ59o/bZ4D9Dze5vf9FvHUGypYtq+ZpaWlqXqJECTW/5ZZbbNnatWvzvC44o8sxAAAAAMCTKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRIt1eAAAA4UrrcnzttdeqYydOnKjm//vf/4K6JgDwsg4dOqi5UzdjJx999FEwloMg4A4tAAAAAMBIFLQAAAAAACNR0AIAAAAAjERBCwAAAAAwEgUtAAAAAMBIdDkGAMAlX3zxhS1r2rSpCysBgPAQaDdjJ8OGDbNlEyZMCMrcCAx3aAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkCloAAAAAgJF8lmVZfg30+UK9FsCRn9s0ZNj/cJPb+1+EMwB3uX0G2P9wk9v7X4QzAHf91RngDi0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEh+dzkGAAAAAKAw4Q4tAAAAAMBIFLQAAAAAACNR0AIAAAAAjERBCwAAAAAwEgUtAAAAAMBIFLQAAAAAACNR0AIAAAAAjERBCwAAAAAwEgUtAAAAAMBIYVXQzp49W3w+n6SlpQX0/7Vq1Uri4+ODupaqVatKnz59gjon8GfY/wh3nAGEM/Y/wh1nwLvCqqD1sp07d0rHjh2lTJkyEhsbK/Hx8fKvf/3L7WUBIZeRkSEjRoyQuLg4iYmJkaZNm8oHH3zg9rKAAsVzAMLR7t275a677pIrr7xSYmNjpVy5ctKyZUtZuXKl20sDCsTp06dlzJgxcuutt0qZMmXE5/PJ7Nmz3V5WgYt0ewHIv7Vr10qHDh3kmmuukSeffFJKlCgh3333nfz4449uLw0IuT59+siSJUtkyJAhUrNmTZk9e7a0a9dOUlJS5IYbbnB7eUDI8RyAcPXDDz/IqVOnpHfv3hIXFydnz56VpUuXSseOHSU5OVn69evn9hKBkDpy5IiMHz9errjiCmnQoIFs2LDB7SW5goLWcCdPnpRevXpJ+/btZcmSJRIRwU13hI9PPvlEFi5cKJMnT5Zhw4aJiEivXr0kPj5ehg8fLlu2bHF5hUBo8RyAcNauXTtp165drmzgwIHSqFEjmTJlCgUtPK9SpUry008/ScWKFWX79u1y3XXXub0kV4T1M9+KFSukffv2EhcXJ1FRUVK9enWZMGGCZGVlqeN37NghCQkJEhMTI9WqVZPp06fbxmRkZMiYMWOkRo0aEhUVJZUrV5bhw4dLRkZGSD6H+fPnyy+//CITJ06UiIgIOXPmjGRnZ4fkWvAWL+z/JUuWSJEiRXK9aImOjpa+ffvK1q1bZf/+/SG5LrzBC2eA5wDklRf2v6ZIkSJSuXJlOX78eIFdE2bywhmIioqSihUrhmRuk4T1HdrZs2dLiRIlZOjQoVKiRAlZv369jB49Wk6ePCmTJ0/ONfbYsWPSrl076dq1q3Tv3l0WL14sAwYMkGLFisn9998vIiLZ2dnSsWNH2bRpk/Tr10/q1q0rX375pUydOlVSU1Nl+fLljmvJzs6Wo0eP+rXuUqVKSdGiRUVEZN26dVKyZEk5cOCAdOrUSVJTU6V48eLSs2dPmTp1qkRHR+ftwYHneWH/f/bZZ1KrVi0pWbJkrjFNmjQREZFdu3ZJ5cqV/X1IEGa8cAZ4DkBeeWH/X3TmzBlJT0+XEydOyDvvvCOrV6+Wbt26BfaAIOx46QyEPSuMzJo1yxIRa9++fZZlWdbZs2dtY5KSkqzY2Fjr3LlzOVliYqIlItYLL7yQk2VkZFgNGza0KlSoYGVmZlqWZVnz5s2zIiIirI8++ijXnNOnT7dExNq8eXNOVqVKFat37945f9+3b58lIn79SUlJyfn/rr76ais2NtaKjY21Bg0aZC1dutQaNGiQJSLW3XffnZ+HCx7jxf1fr149q3Xr1rbPY/fu3ZaIWNOnTw/oMYK3efEM8BwAf3lx//9+3Rc/HhERYXXp0sU6evRoXh4meJiXz4BlWdann35qiYg1a9asAB8Z84X1HdqYmJic/z516pRkZGRIixYtJDk5Wfbs2SMNGjTI+XhkZKQkJSXl/L1YsWKSlJQkAwYMkB07dkizZs3krbfekrp160qdOnXkyJEjOWNbt24tIiIpKSmSkJCgrqVixYp+d2b9/bpOnz4tZ8+elf79++d0tOzcubNkZmZKcnKyjB8/XmrWrOnXvAgvXtj/6enpEhUVZRtz8a5Uenq6X3MiPHnhDPAcgLzywv6/aMiQIdKlSxc5ePCgLF68WLKysiQzM9Ov+RC+vHQGwl1YF7S7d++WUaNGyfr16+XkyZO5PnbixIlcf4+Li5PixYvnymrVqiUiImlpadKsWTPZu3evfPPNN1K+fHn1eocOHXJcS3R0tLRp0ybgz+HiYezevXuu/J577pHk5GTZunUrL2ag8sr+134v5dy5czkfB5x45QyI8ByAwHlh/19Up04dqVOnjoj81hiwbdu20qFDB9m2bZv4fL48zwtv89IZCHdhW9AeP35cEhMTpWTJkjJ+/HipXr26REdHy86dO2XEiBF5aqqRnZ0t9evXlylTpqgf/7Pf5cvKypLDhw/7dZ0yZcpIsWLFROS3A7Z792659NJLc42pUKGCiPz2M//AH3ll/1eqVEkOHDhgG/PTTz+JyG/nA9B45QzwHIC88Mr+d9KlSxdJSkqS1NRUqV27tl/zIrx4/QyEm7AtaDds2CC//vqrLFu2TFq2bJmT79u3Tx1/8OBBOXPmTK7vzqSmpoqISNWqVUVEpHr16vL555/LTTfdFPB3BPfv3y/VqlXza2xKSoq0atVKREQaNWokH3zwgRw4cCDXF+2DBw+KiDh+lwjhzSv7v2HDhpKSkiInT57M1Rhq27ZtOR8HNF45AzwHIC+8sv+dXPx1kz/eZQMu8voZCDdhW9AWKVJEREQsy8rJMjMz5ZVXXlHHX7hwQZKTk2Xo0KE5Y5OTk6V8+fLSqFEjERHp2rWrrFq1Sl577TXbe5+lp6dLdna27ccVLsrrz8537dpVnn32WZk5c2bOz+iLiLz++usSGRnJhofKK/u/S5cu8vzzz8uMGTNy3oc2IyNDZs2aJU2bNqXDMRx55QzwHIC88Mr+P3ToUM5PI1x0/vx5mTt3rsTExMhVV13l15wIP145A/hN2Ba0CQkJUrp0aendu7cMHjxYfD6fzJs3L9fG/r24uDiZNGmSpKWlSa1atWTRokWya9cumTFjRk7r7J49e8rixYulf//+kpKSItdff71kZWXJnj17ZPHixbJmzRpp3LixOn9ef3b+mmuukfvvv1/+7//+Ty5cuCCJiYmyYcMGeeutt2TkyJH8yCVUXtn/TZs2lbvuuktGjhwphw4dkho1asicOXMkLS1NZs6cGfB8CB9eOQM8ByAvvLL/k5KS5OTJk9KyZUu57LLL5Oeff5Y333xT9uzZIy+88IKUKFEi4DkRHrxyBkREpk2bJsePH8/5yZyVK1fKjz/+KCIigwYNklKlSuVpXqO41V7ZDX9s171582arWbNmVkxMjBUXF2cNHz7cWrNmja0ldmJiolWvXj1r+/btVvPmza3o6GirSpUq1rRp02zXyMzMtCZNmmTVq1fPioqKskqXLm01atTIGjdunHXixImccX9s150fmZmZ1tixY60qVapYRYsWtWrUqGFNnTo1KHPDO7y6/9PT061hw4ZZFStWtKKioqzrrrvOev/994MyN7zFq2eA5wD4w4v7f8GCBVabNm2sSy+91IqMjLRKly5ttWnTxlqxYkW+54b3ePEMXJxLHN7i5+Ln6nU+y3L4VgQAAAAAAIVYhNsLAAAAAAAgLyhoAQAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABgp0t+BPp8vlOsA/pTbb5fM/oeb3N7/IpwBuMvtM8D+h5vc3v8inAG466/OAHdoAQAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYKRItxcA/918881q/vDDD6t5x44dbdlzzz2njv3nP/+Z94UBAAAAgAu4QwsAAAAAMBIFLQAAAADASBS0AAAAAAAjUdACAAAAAIxEQQsAAAAAMJLPsizLr4E+X6jXEpYqVapky2655RZ17JQpU9S8VKlSfl/v/Pnzau7UKXnmzJl+zx1Kfm7TkGH/w01u738RzkBBKlGihJq/9tpran733Xer+ccff2zLnJ5fTp486efq3OH2GWD/+6dYsWJqHhUV5fccbdq0UfMxY8aoef369f2e22mOp556yu853OD2/hfhDBQkp3/vcePGqfnYsWNDuJrC4a/OAHdoAQAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkShoAQAAAABGinR7AV7j1J2yR48ean7//ffbskaNGgV1Tb9XpEgRNb/kkktCdk0ULpGR+rF/4IEH1LxmzZp+z3369Gk1f/3119X80KFDtiwjI8Pv6wGmq1Onji1btWqVOrZq1apq7tT9sWnTprasZ8+e6th///vfDitEYeL0HF67dm01T0pKCuVybK6++mo1b9GihZprnXMD7egbyHjtTABuCqRDcWJiYugWYjju0AIAAAAAjERBCwAAAAAwEgUtAAAAAMBIFLQAAAAAACPRFCrInJp5XH/99WoeSEMEp2Y5U6dOVfOHH37Ylh07dkwd++KLL6o5vGfUqFEB5YHQ9rOIyBNPPKHmKSkptmzdunXqWKd8x44dfq4OcE+lSpXUfM2aNbascuXK6tgZM2ao+fjx49X822+/tWVOTeFghgoVKqj5F198UcArKfzS09Nt2bJly1xYCRAcrVq1cnsJhRZ3aAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkCloAAAAAgJEoaAEAAAAARqLdoR/q1Kmj5itWrLBlTt0pA3H06FE1f/DBB9V8+fLlaq511VywYEGe1wXzdO/e3ZY9+eST6lin7tqhdOONN/qViYiMHTtWzXfu3KnmixYtsmUffvihOvbzzz93WCEQmJiYGDV36kavPWe8//776thHH31Uzc+cOaPm7777ri376quv1LGA12jPdbNmzXJhJYCzxMREt5fgCdyhBQAAAAAYiYIWAAAAAGAkCloAAAAAgJEoaAEAAAAARqKgBQAAAAAYyWf52drU5/OFei2ui4zUmz6/9NJLat6/f/98X3P//v227B//+Ic69u2338739UzlRgfe3zN1/+/evduWOXXtDsZj7PQ4FZa5T58+reZO3b8HDBjg99yh5Pb+FzH3DBQ0pz3z73//W8337dtnyxo0aKCOddq/TqpWrWrLDhw4oI49f/58QHMXNLfPQGHZ/1FRUWo+bdo0Nb/vvvvyfc3PPvtMzbXnEqcu3060x9Xp3zo9PV3NnTr3v/nmm7bs8OHDAayu8HB7/4sUnjNgqlatWql5SkqK33OMGzdOzZ3eBcJL/uoMcIcWAAAAAGAkCloAAAAAgJEoaAEAAAAARqKgBQAAAAAYSe+C5HFOTXEGDRqk5sFo/uREa9oBBOrll19Wc22vR0To38fKzs7O9zqc5jh48KCaL1y40JatWrVKHfvhhx+qeVxcnJp369bNljk1XEtKSlLz22+/3Zbdeeed6thdu3ap+YULF9Qc5mvcuLEte/HFF9WxR48eVfOuXbvaskCbPzlJS0sLyjwoPDIyMtR88ODBaj5nzpx8X9Ppa9uOHTtsWfXq1fN9vUA/x1mzZuX7mkCoOTWFQnBwhxYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYKSw7HLs1IUvGN2MP/jgAzV36kILBOKSSy5R85YtW6q5ZVm2zKkT8alTp9TcqUvmtddea8vWrl2rjp0wYYKaB4NTB+WpU6fasp9++kkd++abb6p5pUqVbNnHH3+sjn344YfVPDk5Wc1hPq3ratGiRdWxW7duVXOtUywQqPT0dDXftGlTvud26i5cuXLlfM+tdYF/6KGH1LHB6NgMmGzs2LFuL6HQ4g4tAAAAAMBIFLQAAAAAACNR0AIAAAAAjERBCwAAAAAwEgUtAAAAAMBIPktrg6oN9PlCvZYCs379ejVPTEwMaJ7jx4/bsptuukkdu2vXroDmRm5+btOQKSz7v3fv3mo+c+ZMv+dw+lyGDBmi5uHQodupy3G3bt38nuO9995T8zvuuCNPa/o9t/e/SOE5A25o0qSJmm/ZssWWfffdd+rYxo0bq7lTd3Hk5vYZCIf9P2jQIDWfNGmSmhcrVizf1+zTp48te+ONN/I9r9e4vf9FwuMMhFIg/4YbNmxQ8xtvvDFIqzHPXz1+3KEFAAAAABiJghYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABgp0u0FuOHKK68Myjy9evWyZXQzRrDExcXZsmnTpuV73oMHD6r566+/nu+5TfXzzz/ne45KlSoFYSVwk1PX1tmzZ6t5RIT9e8Lz5s1Txzp1M46OjvZ7HSdPnlRzIBAPP/ywmj/33HNqXrRo0ZCthY7G8JqxY8fme45w7macV9yhBQAAAAAYiYIWAAAAAGAkCloAAAAAgJEoaAEAAAAARvJ8U6iRI0fasiuuuCIoc3/00Uf5niM+Pt6WtWjRIqA5brnlFjXv2LGj33OsWLFCzbt162bLMjMz/Z4Xede6dWtbFhsbm+95nZrTpKen53tuU11yySVq7vP5/J5j48aNwVoOXNK5c2c1r1Onjt9z1KpVS8337dun5pGR9qfhIkWKqGPPnTun5gsXLlTzMWPG2LLz58+rY+FNd955py0bOHCgOjaUzZ+caK/RAvX222+r+Z49e/I9NxAo7esuQo87tAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEgUtAAAAAAAI3mmy7HWKVJE72hsWVZAc7/44otqfubMGVvWoEEDdaxTF9VFixbZsooVK/q/uD8RyOfp1BE5OjraltHluGBcc801tizQvat57bXX8j2HqW6//XY179u3r5oH8ngH498G7mrcuHG+5+jRo4eaO33d1M6jUzfj3r17q/k///lPNX///fdtGd24valGjRpqvmTJkgJeSWCefvppW5adnR3QHE899ZSaL1682JY9+eST6thvv/02oGsCKFy4QwsAAAAAMBIFLQAAAADASBS0AAAAAAAjUdACAAAAAIxEQQsAAAAAMJJnuhwXL15czfv165fvuU+ePKnmrVu3tmVvvPGGOrZcuXJq7vP5bFmg3VIzMjLUvGjRorYsIoLvYYS7hQsXur0E17Rr1y5kc9Ml0xyxsbFq3r59+3zP/cMPP6j5448/ruYLFizwe+6lS5eq+ZYtW9Q8OTnZljVq1Egde/bsWb/XAXMU9u7rWkfjYK35rrvusmVNmjRRx3bu3FnNd+/ebcsuXLiQv4XBM8aOHZvvOcaNG5f/hYA7tAAAAAAAM1HQAgAAAACMREELAAAAADASBS0AAAAAwEg+y8/fvteaFxUmpUqVUvOjR48W8EoCE0hTqHfeeUfNp0+fruZaQ5DKlSsHsDqR0qVL2zKnJlmh5HZjCzf2f0pKii1r0aJFQHPs3LnTljk1xfCa0aNH27InnnhCHRsZqffH0/ZdamqqOrZ58+ZqfuLECacl+s3t/S9S+J8DAtGtWzc1D6RBk4jIgQMHbNmNN96ojg1G0zCt0Z+Ic2NATcWKFdX80KFDeVpTQXH7DBT2/V+yZEk1HzBggC2777771LFOzdKc5o6KirJlZ86cUcceOXJEzbXH1amJptPrvFBq1qyZLdu+fXuBr8Pt/S9S+M9AKLVq1UrNtddpgQrnxzUQf3UGuEMLAAAAADASBS0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADCS3toTBea9996zZf/+97/VsZdccomat2/fXs3j4uL8XseePXvU/MKFC37PgeBKTEy0ZYF2Oty4cWOwllNoxcfHq3lSUpItc+pm7NRlMDMz05b16NFDHRuMbsYoGJUqVQrKPKtXr7ZlwehmDATK6d0HJk2a5Fcm4twBu2rVqmr+t7/9zZb9/PPP6thdu3apuaZhw4Zqft1116n5kCFD1Lx27dp+X9PJ448/bsucuqSfP38+39dD4eTU5TgQ48aNy/9C4Ig7tAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEgUtAAAAAAAI9HlOMicutxNmTJFzZ955hlb1rZtW3XswoUL876w/+e///2vmnfs2FHNz549m+9rIm+0jsaBdjkOdHxh5tTNWOsULiJy6aWX2jKnx0PrZiyid8/cuXOnwwoRbpYsWVKg13PqQutk9+7dtuzUqVPBWg48xqlDsVMeKk4dkZ1yp+eADRs22LIrr7wyoLVor43KlCmjjv3ll18Cmhvm0N51IlDafkTwcIcWAAAAAGAkCloAAAAAgJEoaAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkuhwH2fHjxwMa/9Zbb9mym2++OUirsXv00UfV/LvvvgvZNZE3e/futWU1atRwYSUFa/To0WqelJSk5lo340ANGjRIzV9//fV8z43C59dffw3KPOvXrw/KPH8UGak/Nc+ZMyegeebNm2fL0tPT87QmFA5RUVFq3rlzZzXv37+/Lfvf//6njn3ppZfUfPv27X6uLrSuvvpqNX/sscfUPNCOxpoff/zRljl1xYf5WrVqFVAeCLochxZ3aAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkCloAAAAAgJE80xTK5/O5vQQRESlfvryajxgxQs0jIuzfU8jOzg7omrt371bz+fPn27IPPvggoLnhnvfee8+WPfLIIy6sJP9uv/12NR81apQtu+aaa9SxTo1yLMvyex0PPfSQmtP8KbysXbs2KPOULFnSlh09ejSgOYoWLWrLnBr8ODUmOXDggJo7NfmBuYYNG6bm48aN83uO66+/Xs2dvk5///33av7FF1/YslWrVvm9DhGRkSNH2jKnr+mVK1dW8zJlygR0zUDcc889tuzYsWMhux7cFYzmT3AHd2gBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEbyTJfj06dPq3nLli1tmVPnR6fuqqEUSIfW1NRUNe/QoYOa//DDD3laEwoHbU8H2s1b68IaqNjYWDUvW7asLXvyySfVsX379s33Opw+98zMTDUfNGiQLaObMUScOxF/+OGHap6YmKjmWsfZxx9/XB2rdTMW0TsaL1iwQB3r9DzXvn17Nc/IyFBzmKtChQohm/uSSy5R8wYNGvid9+zZM6Bral/XA3ldFKgff/xRzadNm6bmn376acjWgsLH6Wt9MIwdOzagHIHhDi0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEg+y892coF2Vy3MqlSpouYrV65U83r16oVsLR999JEtW7hwoTp23bp1av7tt98GdU2FUSi7HvrDjf2vdbP86quv1LFlypTxe96lS5cGtI7LL79czZs2bWrLnB6nYPz7Oe3/SZMmqXlKSkq+r1lYuL3/Rbz1HOBE64ovIrJ69Wo1T09Pt2VOZ7R48eJq3qhRI1vm1M24Y8eOar5hwwY19xK3z0Bh2f9O79Lw8MMPF/BKgiOUXY7feecdWzZ69Gh1rNO5LSzc3v8ihecMhFIoH+cbb7xRzcPh63cw/NW/DXdoAQAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkcKyKRTM43ZDhMKy/2vVqqXmAwYMUPMHHnjAlsXGxqpjg/EYB9oUav369bbMqfnTc889l/eFGc7t/S9SeM6AG+Li4tR87ty5tqx169bq2OPHj6v5W2+9ZctefvlldWxhb1wTSm6fgcKy/6OiotQ8MjLS7zm6du2q5ldeeWVAa+nfv78tK126dEBzbNy40ZZt3rxZHet0hqZPn67mGRkZtuzChQv+L64QcXv/ixSeMxBKoXwdhPyhKRQAAAAAwJMoaAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkCloAAAAAgJHocgwjuN3hz9T9X6lSJVvm1IW1YcOG+b7emTNn1Pz1119X80OHDtmyzMzMfK/Da9ze/yLmngF4g9tngP0PN7m9/0U4A3AXXY4BAAAAAJ5EQQsAAAAAMBIFLQAAAADASBS0AAAAAAAjUdACAAAAAIxEl2MYwe0Of+x/uMnt/S/CGYC73D4D7H+4ye39L8IZgLvocgwAAAAA8CQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABjJZ1mW5fYiAAAAAAAIFHdoAQAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkShoAQAAAABGCquCdvbs2eLz+SQtLS2g/69Vq1YSHx8f1LVUrVpV+vTpE9Q5gT/D/ke44wwgnLH/Ee44A94VVgWtV+3du1fuvvtuufzyyyU2Nlbq1Kkj48ePl7Nnz7q9NCDkMjIyZMSIERIXFycxMTHStGlT+eCDD9xeFlAg+vTpIz6fz/HPgQMH3F4iEFI7duyQW2+9VUqWLCmXXHKJtG3bVnbt2uX2soACQx0gEun2ApA/+/fvlyZNmkipUqVk4MCBUqZMGdm6dauMGTNGduzYIStWrHB7iUBI9enTR5YsWSJDhgyRmjVryuzZs6Vdu3aSkpIiN9xwg9vLA0IqKSlJ2rRpkyuzLEv69+8vVatWlcsuu8yllQGht3PnTrnhhhukcuXKMmbMGMnOzpZXXnlFEhMT5ZNPPpHatWu7vUQgpKgDfkNBa7h58+bJ8ePHZdOmTVKvXj0REenXr59kZ2fL3Llz5dixY1K6dGmXVwmExieffCILFy6UyZMny7Bhw0REpFevXhIfHy/Dhw+XLVu2uLxCILSaN28uzZs3z5Vt2rRJzp49K/fee69LqwIKxpNPPikxMTGydetWKVu2rIiI9OjRQ2rVqiWPP/64LF261OUVAqFFHfCbsP6R4xUrVkj79u0lLi5OoqKipHr16jJhwgTJyspSx+/YsUMSEhIkJiZGqlWrJtOnT7eNycjIkDFjxkiNGjUkKipKKleuLMOHD5eMjIyQfA4nT54UEZFLL700V16pUiWJiIiQYsWKheS6MJ8X9v+SJUukSJEi0q9fv5wsOjpa+vbtK1u3bpX9+/eH5LrwBi+cAc38+fPF5/PJPffcU2DXhHm8sP8/+ugjadOmTU4xK/Lb65/ExER599135fTp0yG5LrzBC2eAOuA3YX2Hdvbs2VKiRAkZOnSolChRQtavXy+jR4+WkydPyuTJk3ONPXbsmLRr1066du0q3bt3l8WLF8uAAQOkWLFicv/994uISHZ2tnTs2FE2bdok/fr1k7p168qXX34pU6dOldTUVFm+fLnjWrKzs+Xo0aN+rbtUqVJStGhREfntF9UnTZokffv2lXHjxknZsmVly5Yt8uqrr8rgwYOlePHieXtw4Hle2P+fffaZ1KpVS0qWLJlrTJMmTUREZNeuXVK5cmV/HxKEGS+cgT86f/68LF68WBISEqRq1ap+zYfw5IX9n5GRITExMbYxsbGxkpmZKV999ZU0a9bMz0cE4cYLZ4A64P+xwsisWbMsEbH27dtnWZZlnT171jYmKSnJio2Ntc6dO5eTJSYmWiJivfDCCzlZRkaG1bBhQ6tChQpWZmamZVmWNW/ePCsiIsL66KOPcs05ffp0S0SszZs352RVqlSxevfunfP3ffv2WSLi15+UlJRc80+YMMGKiYnJNeaJJ57I68MEj/Li/q9Xr57VunVr2+exe/duS0Ss6dOnB/QYwdu8eAb+aOXKlZaIWK+88kogDw3CgBf3f/369a1atWpZFy5cyLW2K664whIRa8mSJXl6rOBNXjwDlkUdYFmWFdZ3aH//Xb1Tp05JRkaGtGjRQpKTk2XPnj3SoEGDnI9HRkZKUlJSzt+LFSsmSUlJMmDAANmxY4c0a9ZM3nrrLalbt67UqVNHjhw5kjO2devWIiKSkpIiCQkJ6loqVqzod2fW369L5LfW3y1btpS///3vUrZsWXnvvffk6aeflooVK8rAgQP9mhPhxwv7Pz09XaKiomxjoqOjcz4OOPHCGfij+fPnS9GiRaVr165+zYXw5YX9/9BDD8mAAQOkb9++Mnz4cMnOzpannnpKfvrpJxHhOQB/zgtnQIQ6QCTMf+R49+7dMmrUKFm/fn3Oz6BfdOLEiVx/j4uLs922r1WrloiIpKWlSbNmzWTv3r3yzTffSPny5dXrHTp0yHEt0dHRtk6V/li4cKH069dPUlNT5fLLLxcRkc6dO0t2draMGDFCunfvnut3S4CLvLD/Y2Ji1N9LOXfuXM7HASdeOAO/d/r0aVmxYoXccsstfN3HX/LC/u/fv7/s379fJk+eLHPmzBERkcaNG8vw4cNl4sSJUqJEiYDnRPjwwhmgDvhN2Ba0x48fl8TERClZsqSMHz9eqlevLtHR0bJz504ZMWKEZGdnBzxndna21K9fX6ZMmaJ+/M9+ly8rK0sOHz7s13XKlCmT80ver7zyilxzzTU5m/iijh07yuzZs+Wzzz7L94skeI9X9n+lSpXU99m8+N35uLg4v+ZE+PHKGfi95cuX090YfvHS/p84caIMGzZMdu/eLaVKlZL69evL448/LiL/f8EB/JFXzgB1wG/CtqDdsGGD/Prrr7Js2TJp2bJlTr5v3z51/MGDB+XMmTO5vjuTmpoqIpLTeKN69ery+eefy0033SQ+ny+g9ezfv1+qVavm19iUlBRp1aqViIj88ssvajvu8+fPi4jIhQsXAloHwoNX9n/Dhg0lJSVFTp48masx1LZt23I+Dmi8cgZ+780335QSJUpIx44dA7o2wo/X9n/p0qVzve/4unXr5PLLL5c6deoEtA6ED6+cAeqA34RtQVukSBER+e0N6C/KzMyUV155RR1/4cIFSU5OlqFDh+aMTU5OlvLly0ujRo1ERKRr166yatUqee2113K9jYjIb7/HkZ2d7dhtLK8/O1+rVi1Zu3atpKam5vpO5IIFCyQiIkKuvvpqv+ZEePHK/u/SpYs8//zzMmPGjJz3oc3IyJBZs2ZJ06ZN6XAMR145AxcdPnxY1q1bJ927d5fY2Fi/5kH48tr+/71FixbJp59+Ks8//7xERIT1u1PiT3jlDFAH/CZsC9qEhAQpXbq09O7dWwYPHiw+n0/mzZuXa2P/XlxcnEyaNEnS0tKkVq1asmjRItm1a5fMmDEjp3V2z549ZfHixdK/f39JSUmR66+/XrKysmTPnj2yePFiWbNmjTRu3FidP68/O//YY4/J6tWrpUWLFjJw4EApW7asvPvuu7J69Wp54IEH+JFLqLyy/5s2bSp33XWXjBw5Ug4dOiQ1atSQOXPmSFpamsycOTPg+RA+vHIGLlq0aJFcuHCBHzeGX7yy/zdu3Cjjx4+Xtm3bStmyZeXjjz+WWbNmya233iqPPPJIwPMhfHjlDFAH/D9utVd2wx/bdW/evNlq1qyZFRMTY8XFxVnDhw+31qxZY2uJnZiYaNWrV8/avn271bx5cys6OtqqUqWKNW3aNNs1MjMzrUmTJln16tWzoqKirNKlS1uNGjWyxo0bZ504cSJn3B/bdefHtm3brNtuu82qWLGiVbRoUatWrVrWxIkTrfPnzwdlfniDV/d/enq6NWzYMKtixYpWVFSUdd1111nvv/9+UOaGt3j1DFiWZTVr1syqUKFCrrcvAX7Pi/v/22+/tdq2bWuVK1fOioqKsurUqWM988wzVkZGRr7nhvd48QxYFnWAZVmWz7IcvhUBAAAAAEAhxi8XAAAAAACMREELAAAAADASBS0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEiR/g70+XyhXAfwp9x+u2T2P9zk9v4X4QzAXW6fAfY/3OT2/hfhDMBdf3UGuEMLAAAAADASBS0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEgUtAAAAAAAI1HQAgAAAACMFOn2AsLdP/7xD1v2wgsvqGPvu+8+NZ8zZ05Q1wQAAAAAJuAOLQAAAADASBS0AAAAAAAjUdACAAAAAIxEQQsAAAAAMBIFLQAAAADASHQ5LiCrV69W85tuusmWbdiwQR27ZMmSYC4JKDBOHbpHjRply6pVq6aO9fl8am5Zlppr5+Xpp59Wx+7atUvNAQAAULhxhxYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYCSf5dQi9I8DHTqMhrOyZcvaspUrV6pjmzRpoubHjh2zZTfccIM69r///W8Aq/MWP7dpyLD/7SZNmmTLHnnkEXVsZKTeUL2gH1ftvImItGnTRs0LS/djt/e/CGcA7nL7DLD/4Sa3978IZ6AgXXLJJWo+YMCAgOYZP368LYuKilLHjhgxQs2fe+65gK4ZKn91BrhDCwAAAAAwEgUtAAAAAMBIFLQAAAAAACNR0AIAAAAAjERTKD84NWl6+eWXbVmDBg3UsXPmzFHzwYMH27JTp04FsLrw4HZDhHDe/zt27FDzq6++2pZFRJj5PbJ58+apeZ8+fQp2IQ7c3v8i4X0G4D63zwD73z+NGjVS806dOql5+fLlbdmdd97p91gRkW+++UbNly1bZsueeeYZdezZs2fVvLBwe/+LcAby64477lDz4cOH27LatWurY0uXLh3UNf3e+fPn1fzFF1+0Za+//ro69ttvvw3mknKhKRQAAAAAwJMoaAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkCloAAAAAgJHCssuxUyfWZ599Vs0HDhyo5pGRkbbs0UcfVcdOmzZNzQtD5zoTuP04eWn/B8qpg2StWrUKeCWhc+bMGTXXOpx/8cUXoV6Ojdv7X8SdM6Bds2bNmurYzp07q3lcXJzf1/v73/+u5pUqVVLzQB4Tp3/DNWvW2LK9e/eqY5966ik1P3TokN/rMJXbZyCcnwNatmyp5iNHjrRlbdu2Vcc6/ftpj2sgYwMd36tXL3Xsm2++qeaFhdv7XyS8z4CTp59+2pY5dTOuVq2amkdFRQV1Tb/3/vvv2zKnbuFOHco1X3/9tZrXr1/f7zkCRZdjAAAAAIAnUdACAAAAAIxEQQsAAAAAMBIFLQAAAADASBS0AAAAAAAj2dv0ekzFihVt2bhx49SxDz74oJrv379fzceMGWPLZs+e7f/iwlx0dLQtO3funAsrwZ9Zt26dmnupy3Hx4sXVvFWrVrbMjS7H4apo0aK2zKnrdig5dVcMRudRrSusU6fYEiVKqPkTTzyh5j/99FPeFwbPcvp6N3fuXDW/88471Vzb/4F2wg1kfDDmdvoc165dq+aHDx8O6JowW506ddR8+vTpaq69E0Kg+/Ts2bO27Msvv1THvvPOO2q+adMmNd+6dastGzx4sDo2kC7H6enpfo8tKNyhBQAAAAAYiYIWAAAAAGAkCloAAAAAgJEoaAEAAAAARvJMU6hLL71Uzd9//31bdvXVV6tjDxw4oOa33HKLmu/Zs8fP1YW3Ll26qLnWyOSaa64J9XLg4Morr1Tzzp07F/BKCp7WlEFE5OOPPy7gleD3nL5Wh4pTk6czZ86oeVpami2rXbt2QNfUGl856d27t5r/73//U/OxY8cGtBaEh3/+859qfscdd6h5MJqiLVu2TM2XL19uy5yaUAXSnMqJ01inuWfMmOH33DDH8OHD1bxfv35qXq1aNb/nPn36tJo//vjjav7111/bspSUFL+v92dKlSply4YMGRLQHOfPn7dlkyZNyuuSQoY7tAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEgUtAAAAAAAI3mmy/HTTz+t5lqXzC+//FIde91116l5ZmZm3hdmOK0DZ6NGjdSx06ZNU/OrrrpKzZOSkvK+MOSZU9fpl156Sc0rVqwYyuXk21dffaXm8fHxfs8RGxur5s2aNbNln3zyid/zIn8+++wzW7Zo0SJ1rNO/t9ahMTk5WR37/fffq/m6deuclui3mJgYNX/77bdt2c033xzQ3E6d+J999llbdu7cuYDmhtm07r2jRo1Sx2ZnZ6u5z+dTc23vOj2/BMKpC6vTOpwEOh7eVLVqVVsWjG7GIiKrVq2yZVOmTFHHBqtzcSAWL15syy6//PKA5tA6Gi9dujTPawoV7tACAAAAAIxEQQsAAAAAMBIFLQAAAADASBS0AAAAAAAjUdACAAAAAIzksyzL8mtgIe8W9/e//13NX375ZVtWoUIFdeyHH36o5k899ZSau9GxTFOqVCk1L126tC2799571bHdunVTc60zp9P1Xn/9dTV36kr6+eefq7nGz20aMoV9/zvRuvu999576tg6deqEeDV28+fPt2VaR70/49SZc/Xq1bYs0O5+3377rS2rXbt2QHMEg9v7X8TcM1DY/e1vf7NlTp20q1evHtDcY8aMsWVOz2eFndtnwNT9/+mnn9qya6+9Vh3r9BgvX75czXv16mXLzp496//iHGhrFgl83dq/mdNYp27+R44cUfOC5vb+FzH3DGjvVrB58+aA5vjPf/6j5rfddpsty8rKCmjuYLjxxhvVXOvCXKxYMXVsWlqammtd9LXXRqH2V2eAO7QAAAAAACNR0AIAAAAAjERBCwAAAAAwEgUtAAAAAMBIFLQAAAAAACNFur2AYFm6dKmaf/fdd7bslVdeUcc6dQlr3Lixms+ZM8eWPffcc+rYH3/8Uc2LFy9uy7p06aKO1ToKiohUq1ZNzbUOt/v371fHrl+/Xs2/+eYbW/Z///d/6tjC0g0Q/78VK1bYMje6GR89elTNJ0+ebMu++uqroFxz48aNtuyee+4JaI4rr7zSlvXs2VMdO2/evIDmBkREjh8/bsucOu4H2uW4ffv2tuyZZ55Rx7rRmRPuCLRbrdP4unXr+j1Hp06d1Fx7hwqnTvLBWPd1112njuX1C/6M09fkgv66+cILL6j5oEGD1LxIkSK2zKlDsdaxWUTk+++/93N17uIOLQAAAADASBS0AAAAAAAjUdACAAAAAIxEQQsAAAAAMJLPsizLr4EB/jJ+YRYZqffCGjFihJr369dPzStXruz3Nd98800179ixoy275JJL1LFa8xARkZkzZ6q51ijr448/dlhh4ebnNg2Zwr7/u3btquZaoyKn/R8M27dvV/OxY8eq+erVq/N9zSpVqqj5J598YsvKlSsX0Nxnz561ZU5NtQ4cOBDQ3IFwe/+LFP4z4CX33Xefmr/++uv5njs6OlrNz58/n++5Q8ntM2Dq/v/0009t2bXXXquOdXqMnT53bXwgY53GB2MdIiLLly+3ZU7NNbWv9YWJ2/tfxNwzUKxYMVvm1Ei2Xbt2an7q1Ck1b9u2rS3TXnv8mR49etgyp+Z9pUqVUnOtwayT0aNHq/nEiRP9nsMNf3UGuEMLAAAAADASBS0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADBS6NqdFmIlSpRQ87feekvNq1evruZ9+vTx+5r33nuv32Od1jF16lQ1N7VzMQJ39913q/m4cePUPJQdjefPn2/LHnroIXWsU4fAYKhdu7aaB9rRWJOVlWXLQtnNGAimFStW2DJtT8O79uzZY8saNWoU0ByBdLcNtBNuMOY+cuSImnfp0iWgtcCbMjMzbdmXX36pjnXqcuz07iPau6NMnz5dHTtkyBA1v+GGG2yZU53i5Pvvv1dzrfb4/PPPA5rbFNyhBQAAAAAYiYIWAAAAAGAkCloAAAAAgJEoaAEAAAAARqKgBQAAAAAYyfNdjps2bWrLXnrpJXVskyZN1Dw7O1vNjx07ZssWLVqkji1fvrya33nnnbasVatW6tjx48erOcLHFVdcoeY1atQI2TUfeOABNV+yZIktC2U3Y60ToIjI7NmzQ3bNuXPnhmxuwEl0dHRQ5vn5559tmdPzGbypZ8+etszpHRPq1q2r5t98843f13OaY86cOX7PYVmW32NFRJ5++umAxgNPPfWUmsfHx6t5+/bt1bxTp05+ZYE6f/68mj/zzDNq/sYbb6j5d999l++1mII7tAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEiebwr1/PPP2zKn5k+//PKLmj/77LNq7tRcKhCjR4+2ZWPHjlXHrl+/Xs3btm2r5p9//nme14Xwc/z4cTVPSUlR81A1gHJq/rR48WI1v/TSS/N9TafP5cUXX8z33ECgHnzwwaDMc+jQoaDMA2/ZuXNnQHkgtEaXIiI+ny+gXLN27Vo1D8ZrMXhXsWLFbFnNmjXVsXXq1An1cmy0pmbbt29Xx65YsSLUyzEWd2gBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEbyTJfjRx99VM2bN29uy7Kzs9WxTp0l33333bwv7C9MnDjRlt10003q2BYtWqh5jx491JwuxwjEe++9p+ZpaWn5nrtKlSpqXrt2bVs2e/ZsdWwwuhk7yczMVPPvv/8+ZNcEREQqVKhgy0qXLh3QHE4d+l977bU8rQnIq8cff1zNLcvyew6nsT179szTmhAe4uPj1XzUqFG27K677grKNS9cuGDLIiMDK60+/PBDW7Zu3bo8rylccYcWAAAAAGAkCloAAAAAgJEoaAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkz3Q57tSpk5pHRNhr9kWLFqljQ9nN2ElWVpYtO3/+fEBz3HvvvWr+3HPP2bLDhw8HNDfCR/HixdW8aNGial6sWDFbdv3116tj582bp+blypXzc3WhdezYMbeXgDDVq1cvW3bFFVcENMeOHTvU/MCBA3laE+AP7eu6z+cLaA5t/IwZM9SxR44cCWhuhJe+ffuqeSAdjTMyMtR88uTJan78+HFb9vzzz/t9PRGR1q1b2zK6HAeOO7QAAAAAACNR0AIAAAAAjERBCwAAAAAwEgUtAAAAAMBIFLQAAAAAACN5psvxd999p+Za11VTO5o6dQ/84osv1JyOxgiEU6fw+fPnq3mpUqVs2U033RTMJQXdrFmz1DzQroRAoLTzIiIycOBAv+fIzMxUc6cOnEAw1KlTR8215wzLstSxTrnWufi1117zf3EIOy+//LKa9+/f3+85nF7XDBgwQM1Pnz6t5o888ojf13Ryyy232LLHH3883/OGG+7QAgAAAACMREELAAAAADASBS0AAAAAwEgUtAAAAAAAI3mmKdS2bdvUvFevXrasfPnyoV6OTdOmTdW8R48etiwxMVEde+LECTV/6qmn8r4w4C907tzZ7SX8qX379qm51uhpw4YN6tg9e/YEc0mATe/evdW8cuXKfs+xcePGgHIgGG677TY1j42NtWVOzSudvPnmm7Zs586dAc2B8NKtWzc1j4jQ79Ht2rXLljk1kDpz5kye15VXX375ZYFf04u4QwsAAAAAMBIFLQAAAADASBS0AAAAAAAjUdACAAAAAIxEQQsAAAAAMJJnuhzPmTNHzW+++WZbduedd6pj//Of/6j5+vXr1bxYsWK27O6771bHVq9eXc21rmyHDh1Sxzp1dtu0aZOaw3uc9uivv/6q5mXKlLFlgXahLCycuhnfeuutav7tt9+GcjmA6tprr1XzYHSjf+edd/I9BxCoTp06qbllWX7P4TT26aefzsuSAL9lZGTYsmB1M65atWq+53jjjTfyvxBwhxYAAAAAYCYKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYCTPdDk+e/asmvfo0cOW9erVSx07fPhwNZ8wYULeF/b/OHUx++yzz2zZ7Nmz1bHHjh3L9zpgth07dqh5hQoV1HzgwIG2bOzYserY0qVL53ldF2VnZ6u5U2flCxcu2DKn/f/888+rOd2MUZjccsstal68eHG/5zh48KCaz5w5M09rAvyRlJSk5i1btlRz7eu99s4NIvprMRGRI0eO+Lk64DdTp05V89GjR6t57dq1bVnXrl3VsV999ZWaO31dHzRokJpr1qxZo+a7du3yew444w4tAAAAAMBIFLQAAAAAACNR0AIAAAAAjERBCwAAAAAwks+yLMuvgQ5NXYCC4Oc2DRkv7f+GDRuqefv27dX8kUceUfPDhw/bsqeeekodGxUVpeYbNmywZWlpaerYcOb2/hfx1hkIlu7du9uy119/XR0bHR3t97xt2rRR85SUFL/n8Bq3z4CX9n/58uXVfNWqVWp+7bXXqrn2b+L0OF133XVqvnPnTjVHbm7vf5HCfwZGjhyp5mPGjLFlRYsWDdk6MjIy1LxJkyZq7tSICrn91RngDi0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEh0OYYR3O7wx/6Hm9ze/yLhfQaKFCmi5m+99ZYtu+OOOwKae8uWLbasZcuW6tjCsA/c4vbn7qX937hxYzXftm2bmkdE6Pc+srOzbZlT1+LbbrtNzY8cOaLmyM3t/S9i7hnQutH/85//VMfGx8cHNPemTZts2XPPPaeOfe+99wKaG7nR5RgAAAAA4EkUtAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEh0OYYR3O7wx/6Hm9ze/yLhfQZKly6t5sHo0Lp582Zb5tTlOJy5fQa8tP9jY2PV3KnL8VVXXaXmy5Yts2UDBgxQx9LNOH/c3v8i3joDMA9djgEAAAAAnkRBCwAAAAAwEgUtAAAAAMBIFLQAAAAAACNR0AIAAAAAjESXYxjB7Q5/7H+4ye39LxLeZ6BIkSJqPnr0aFs2atQodWxKSoqa33fffbZs//79AawuPLh9BsJ5/8N9bu9/Ec4A3EWXYwAAAACAJ1HQAgAAAACMREELAAAAADASBS0AAAAAwEg0hYIR3G6IwP6Hm9ze/yKcAbjL7TPA/oeb3N7/IpwBuIumUAAAAAAAT6KgBQAAAAAYiYIWAAAAAGAkCloAAAAAgJEoaAEAAAAARvK7yzEAAAAAAIUJd2gBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEYKq4J29uzZ4vP5JC0tLaD/r1WrVhIfHx/UtVStWlX69OkT1DmBP8P+R7jjDCCcsf8R7jgD3hVWBW04mDhxovh8vqAfPKAw2rBhg/h8PvXPxx9/7PbygJDbvXu33HXXXXLllVdKbGyslCtXTlq2bCkrV650e2lAgeM1EMJRRkaGjBgxQuLi4iQmJkaaNm0qH3zwgdvLKlCRbi8AwfPjjz/K008/LcWLF3d7KUCBGjx4sFx33XW5sho1ari0GqDg/PDDD3Lq1Cnp3bu3xMXFydmzZ2Xp0qXSsWNHSU5Oln79+rm9RKBA8BoI4apPnz6yZMkSGTJkiNSsWVNmz54t7dq1k5SUFLnhhhvcXl6BoKD1kGHDhkmzZs0kKytLjhw54vZygALTokUL6dKli9vLAApcu3btpF27drmygQMHSqNGjWTKlCkUtAgbvAZCOPrkk09k4cKFMnnyZBk2bJiIiPTq1Uvi4+Nl+PDhsmXLFpdXWDDC+keOV6xYIe3bt5e4uDiJioqS6tWry4QJEyQrK0sdv2PHDklISJCYmBipVq2aTJ8+3TYmIyNDxowZIzVq1JCoqCipXLmyDB8+XDIyMkL6uWzcuFGWLFkiL774YkivA+/w0v4XETl16pRcuHAh5NeBd3jtDFxUpEgRqVy5shw/frzArgnzeGn/8xoIeeGFM7BkyRIpUqRIrm9eRkdHS9++fWXr1q2yf//+kFy3sAnrO7SzZ8+WEiVKyNChQ6VEiRKyfv16GT16tJw8eVImT56ca+yxY8ekXbt20rVrV+nevbssXrxYBgwYIMWKFZP7779fRESys7OlY8eOsmnTJunXr5/UrVtXvvzyS5k6daqkpqbK8uXLHdeSnZ0tR48e9WvdpUqVkqJFi+b8PSsrSwYNGiQPPPCA1K9fP/AHAmHJK/tfROS+++6T06dPS5EiRaRFixYyefJkady4cWAPCMKOl87AmTNnJD09XU6cOCHvvPOOrF69Wrp16xbYA4Kw4pX9z2sg5JUXzsBnn30mtWrVkpIlS+Ya06RJExER2bVrl1SuXNnfh8RcVhiZNWuWJSLWvn37LMuyrLNnz9rGJCUlWbGxsda5c+dyssTEREtErBdeeCEny8jIsBo2bGhVqFDByszMtCzLsubNm2dFRERYH330Ua45p0+fbomItXnz5pysSpUqVu/evXP+vm/fPktE/PqTkpKSa/5p06ZZpUqVsg4dOpSz3nr16uXpMYJ3eXH/b9682fr73/9uzZw501qxYoX1zDPPWGXLlrWio6OtnTt35ufhggd58Qz8ft0XPx4REWF16dLFOnr0aF4eJniUV/c/r4HgLy+egXr16lmtW7e2fR67d++2RMSaPn16QI+RqcL6Dm1MTEzOf586dUoyMjKkRYsWkpycLHv27JEGDRrkfDwyMlKSkpJy/l6sWDFJSkqSAQMGyI4dO6RZs2by1ltvSd26daVOnTq5fn+jdevWIiKSkpIiCQkJ6loqVqzod0ey36/r119/ldGjR8uTTz4p5cuX9+8TB8Qb+z8hISHXnB07dpQuXbrI1VdfLSNHjpT333/frzkRnrxwBi4aMmSIdOnSRQ4ePCiLFy+WrKwsyczM9Gs+hCcv7H9eAyE/vHAG0tPTJSoqyjYmOjo65+PhIKwL2t27d8uoUaNk/fr1cvLkyVwfO3HiRK6/x8XF2Trn1apVS0RE0tLSpFmzZrJ371755ptvHL+oHjp0yHEt0dHR0qZNm4A/h1GjRkmZMmVk0KBBAf+/CG9e2P+aGjVqyB133CHLli2TrKwsKVKkSFDmhfd46QzUqVNH6tSpIyK/NQRp27atdOjQQbZt2yY+ny/P88K7vLD/eQ2E/PDCGYiJiVF/P/fcuXM5Hw8HYVvQHj9+XBITE6VkyZIyfvx4qV69ukRHR8vOnTtlxIgRkp2dHfCc2dnZUr9+fZkyZYr68T/7GfasrCw5fPiwX9cpU6aMFCtWTPbu3SszZsyQF198UQ4ePJjz8XPnzsn58+clLS1NSpYsKWXKlAnsE4HneWH//5nKlStLZmamnDlzxvZ7JYCI989Aly5dJCkpSVJTU6V27dp+zYvw4YX9z2sg5IcXzoCISKVKleTAgQO2MT/99JOI/FaIh4OwLWg3bNggv/76qyxbtkxatmyZk+/bt08df/DgQTlz5kyu786kpqaKiEjVqlVFRKR69ery+eefy0033RTwd8T3798v1apV82tsSkqKtGrVSg4cOCDZ2dkyePBgGTx4sG1ctWrV5JFHHqHrH2y8sP//zPfffy/R0dFSokSJgNaB8OH1M3Dxx8z+eJcBEPHG/uc1EPLDC2dARKRhw4aSkpIiJ0+ezPUN/G3btuV8PByEbUF78ccQLcvKyTIzM+WVV15Rx1+4cEGSk5Nl6NChOWOTk5OlfPny0qhRIxER6dq1q6xatUpee+0123v/paenS3Z2tuMbfuflZ+fj4+Pl7bfftn181KhRcurUKXnppZekevXqfs2J8OKF/S8icvjwYduP9nz++efyzjvvyG233SYREWH9zmT4E145A4cOHZIKFSrk+vj58+dl7ty5EhMTI1dddZVfcyK8eGH/8xoI+eGFMyDy20/jPP/88zJjxoyc96HNyMiQWbNmSdOmTcOjw7GEcUGbkJAgpUuXlt69e8vgwYPF5/PJvHnzcm3s34uLi5NJkyZJWlqa1KpVSxYtWiS7du2SGTNm5LTO7tmzpyxevFj69+8vKSkpcv3110tWVpbs2bNHFi9eLGvWrHF8K5G8/Ox8uXLlpFOnTrb84ncjtY8BIt7Y/yIi3bp1k5iYGElISJAKFSrI119/LTNmzJDY2Fh59tlnA54P4cMrZyApKUlOnjwpLVu2lMsuu0x+/vlnefPNN2XPnj3ywgsv8FMKUHlh//MaCPnhhTMgItK0aVO56667ZOTIkXLo0CGpUaOGzJkzR9LS0mTmzJkBz2cst9oru+GP7bo3b95sNWvWzIqJibHi4uKs4cOHW2vWrLG1xL7YAn779u1W8+bNrejoaKtKlSrWtGnTbNfIzMy0Jk2aZNWrV8+KioqySpcubTVq1MgaN26cdeLEiZxxf2zXHUy0rIfGi/v/pZdespo0aWKVKVPGioyMtCpVqmT16NHD2rt3b77nhvd48QwsWLDAatOmjXXppZdakZGRVunSpa02bdpYK1asyPfc8BYv7n8Nr4HgxKtnID093Ro2bJhVsWJFKyoqyrruuuus999/Pyhzm8JnWQ7figAAAAAAoBDjF8wAAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgpEh/B/p8vlCuA/hTbr9dMvsfbnJ7/4twBuAut88A+x9ucnv/i3AG4K6/OgPcoQUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGCnS7QUAAFCYtW3bVs179uxpy+6991517K5du9Q8LS3NlnXu3NnvtQEAEO64QwsAAAAAMBIFLQAAAADASBS0AAAAAAAjUdACAAAAAIxEQQsAAAAAMJLPsizLr4E+X6jX4mljx45V8w0bNviVhTs/t2nIsP/hJrf3v0h4n4GtW7eqeZMmTfI999mzZ21Znz591LFLly7N9/VM5fYZCOf9X9BefvllNX/zzTfV/OOPPw7lcgoFt/e/CGegMBs8eLCa16lTx5YlJSUFNHdEhP3eZ40aNdSx3333XUBzB+KvzgB3aAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkCloAAAAAgJFoCpUPrVq1smVjxozxe6yTwv5YO30uTrlTk6tAml+53RChsP+bwNvc3v8i4X0Gfv75ZzUvX758vufWHtf//Oc/6tibb74539czldtnwEv7v169emr+z3/+U82dmpEtX74832t59NFHbdlzzz2njnVq/nT99dfnex2Fndv7X8RbZ6Aw0Z5HXnvtNXVs3bp11dypSVMw9o327/7444+rYydNmpTv6zmhKRQAAAAAwJMoaAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkCloAAAAAgJEi3V6AybSuvoF0MzaB9vmkpKQENIdT52c65iEYLrnkEjUvXry4LUtPT1fHxsTE5Hsdx44dU/OMjIx8zw13vfDCC2r+7LPPhuR6xYoVU/PISP0p+8KFCyFZB7wpPj5ezXv06KHmnTp1UnOt4+qPP/4Y0Fq0DsUREfq9lmbNmql5oO+wALihb9++at6vXz9b1qhRo1AvJ1+WLFni9hJsuEMLAAAAADASBS0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADASXY5/x6lTnlPu1L03EOPGjcv3HIEIh88R7hkxYoSa33jjjfme26krds2aNdW8SpUqtuzgwYPq2Msuu0zNLcvyc3UiO3fuVPPrrrvO7zlQOE2dOlXNb731VlsWjE73N9xwg5rXqVNHzb/66qt8XxNwUqJECTV/8cUXbVmXLl1Ctg6n7sfDhw9Xc7ocI9TKly9vy5544gl17KBBg9Q8kNcZcMYdWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYKSwbAoVaAOkYDT5cGqMNHbs2HzPHYhQfo5ODRgK+nOEe5555hm3l/CnnJo/OTWcCkTFihXzPQcKp/j4eDWvV69eSK63fft2Nf/+++9Dcj2El6uvvjoo8wTyddOpoVN0dHS+13Hu3Ll8zwH8mV69eqn5Y489Zsvq1q2b7+u98847aq41YhMR2bhxo99z9+3bV82Tk5P9nqMw4g4tAAAAAMBIFLQAAAAAACNR0AIAAAAAjERBCwAAAAAwEgUtAAAAAMBInu9yrHXYTUxMVMea2s04kK7NoexmfOONN+Z7bhQ+l1xyiZp/+OGHfs9hWZaap6am2rL09HR17ObNm9U8LS1NzatWrer32EB8/fXXar5ly5Z8zw13OXUtXrNmjZqXK1cuJOto0KCBmo8fP17NR48ereZnz54N2prgHY0aNSrwazp1Vr7tttvyPffChQvzPQfCi1OH7okTJ6r5P/7xDzUvWrSo39f84Ycf1Lx79+627Msvv1THBvo1Xat3pk6dGtAcc+fOtWX/+9//ApqjIHCHFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJM93OdY6/YZSKLsZO3H6HIPR0VhDN2NvqlChgppPnjxZzZ06sWoeeOABNde6Uzp1OQZCzakTa6i6GTtx6pzp1GmzcePGat6pUydbdvz48bwuC8jl448/tmVNmjRRx86ZMyfUywH81r9/fzUfPnx4yK45a9YsNd+2bVvIrqk9Z8TGxqpjDx8+rOaTJk2yZefPn8/fwkKAO7QAAAAAACNR0AIAAAAAjERBCwAAAAAwEgUtAAAAAMBIFLQAAAAAACMZ1+XYqXNvQXczFgldt1+nzzElJSUk1xMR2bBhg5qPGzcuZNdE4fLkk0+qeY8ePfI9t1NXvejoaFtGl2MgMC1atFDz5ORkW9atW7dQLweF3FdffaXmt9xyS0DzPPfcc8FYTr5lZma6vQQYpnbt2iGbe8mSJWo+YcKEkF0zMTFRzbXnBqduxk7nf8+ePXlfWAHiDi0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADCSz7Isy6+BPl+o1+KXsWPHqnkom0I5NUwKRlMorQFUKJs/OXFq/qQ93k7/Bk55MPi5TUOmsOz/YNAaxYiIPPjggwW8Et26devUfPz48Wq+adOmUC6nUHB7/4t46ww46d69u5q/8cYbIbum9rgG6987IyPDliUkJKhjd+3aFZRrhorbZ8BL+9+pMdjChQsLeCWBcWoY6NR00Evc3v8i3joDTp/LhQsXAprn22+/tWWhbDjlxGl/ZGdn27KZM2eqY/v16xfUNQXbX50B7tACAAAAAIxEQQsAAAAAMBIFLQAAAADASBS0AAAAAAAjUdACAAAAAIwU6fYCApWYmBiyuQPp9BsorZuxSGi7MwfCaR1a7tT12elzdBqP0GvTpo0tc+pmHIwuik6dA7/66is1r1atmi276aab1LGXX365mjdt2lTNT506peaAk507d6r50KFD/Z5j2bJlal62bFk1175u3n777erYFi1aqHnRokXVPDo62pY5nZfC3uUYwfPZZ5+p+TfffKPmdevWDeVy/LZ37163lwCPGDVqlJo7vQ46fPiwmg8ePDhoa/q94sWLq/mLL76o5lo3YxH99feQIUPyuKrCjTu0AAAAAAAjUdACAAAAAIxEQQsAAAAAMBIFLQAAAADASBS0AAAAAAAj+Sw/W5s6dS8taE4dh0PZLdip+7HWcdmp06+X3HjjjWoeym7GwejAmx+FZf8Hw4ABA9T8gQceUPN3331XzZcuXer3Nb/++ms117q2rlu3Th2bmZmp5ldccYWaO3UlNJHb+1/EW2fAVO+9956a33rrrX7PkZCQoObbtm3L05oKittnIBz2v9PX0o4dO6p5jRo1bFmPHj3UsT/99JOax8fH+7k6kfT0dDWPjY31ew5Tub3/Rcw9A8WKFbNlM2bMUMc67d+5c+eq+f3335/3hf2J559/Xs2dOhQ7/dvccccdtszpNV1h91dngDu0AAAAAAAjUdACAAAAAIxEQQsAAAAAMBIFLQAAAADASMY1hUpJSVHzcGjGFEpOja+0Rk+hbP7kxO2GCIVl/3vNsGHDbNmkSZPUsbt371bzpk2bqrlTAxETub3/RTgDhYHT81/Lli39noOmUHnD/s8fp2Y2U6dO9XsOmkK5y9QzoDUv27NnT0Bz1K5dW82/++67PK3p96666ipbtnLlSnVslSpV1Pyjjz5S806dOtmyEydO+L+4QoSmUAAAAAAAT6KgBQAAAAAYiYIWAAAAAGAkCloAAAAAgJEoaAEAAAAARop0ewGBuvHGG9U8nLsfO3Uo1owdOzZ0C0FQxcfH27IFCxaoY1evXq3mw4cPD+qagq1Hjx62zNROijBHqVKl1Dw6OlrNDx8+rObZ2dn5XkuZMmVs2YsvvqiObdGiRUBzX7hwwZadP38+oDkAwGvceJ3RoEEDNV+7dq0tK1eunDp248aNau5UG4UT7tACAAAAAIxEQQsAAAAAMBIFLQAAAADASBS0AAAAAAAjUdACAAAAAIxkXJdjJ04dvrSuvomJiepYNzoib9iwwZZ9+OGH6lg6FIeXFStW2LKqVauqY4cNG5bv68XExKh5+fLl1bx06dK27O9//7s69oEHHlDzsmXL2jLLstSxnTp1UvP09HQ1B5y8+uqrat6tWzc1HzlypJo/99xztkw7FyIitWvXVvPHH3/clrVv314dG6gdO3bYsp07dwZlbiAQK1euVPOpU6f6PYdTF/Kbb75ZzT/44AO/54Z3Pfnkk7bM6XXG3Llz1fx///tfvteRlJSk5trroFWrVqljtXeGwG+4QwsAAAAAMBIFLQAAAADASBS0AAAAAAAjUdACAAAAAIxEQQsAAAAAMJJnuhw70ToDO3ULdqPL8bhx42yZ1vkY4UfrUOrU5XjOnDlq/tNPP9my9evXq2OdOquWK1dOzbVurj6fTx3r1FEwOzvblm3ZskUde+zYMTUHQm3ixIlqfuedd9oypy7HNWvWVHPtzDidFydffPGFmk+ePDmgeYBQiYqKyvccTs8vTmcO4aVx48Zqfsstt/g9x4kTJ9T8/Pnzal6sWDFbdsUVV6hjnbocnz592pZNmjQpoPWBO7QAAAAAAENR0AIAAAAAjERBCwAAAAAwEgUtAAAAAMBInm8KpTV6GjNmTMiu59TQSWv+9GfjgaNHj/o9tnz58n7nDRo0UMcG2ogmEE4Nnfr27WvLVqxYEbJ1AHkREaF/77dJkyYFuo4LFy6o+WOPPabm69atC+VyAL8F8nwG5IXT6yCnxpbBoDWA2rNnT0BzDBkyxJZt2rQpr0sKW9yhBQAAAAAYiYIWAAAAAGAkCloAAAAAgJEoaAEAAAAARqKgBQAAAAAYyfNdjlNSUgr0eh9++KGa080Ygerfv78tGzlypDr29ttvV/Mrr7zSliUkJKhjg9Hl+M0331TzJUuWqHl6enq+rwmYzOfz2bLTp0+rY3v27KnmdDNGYXfkyBE1114zJSYmBjS3UxdyQET/Ghvo2GeeeUbNhw8f7vfc3bp1U3On10cIDF8FAAAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkShoAQAAAABG8kyX41atWhX4NbXOxWPHji3wdSB8HDt2TM3nzZtXwCsBzLVs2TI1d+pCGQz79u1T8y1bttiyKVOmqGN37doVzCUBritSpEi+5+jdu7eaL1y4MN9zw3yBvIODUyf52NhYNdc60m/cuFEdSzfj0OIOLQAAAADASBS0AAAAAAAjUdACAAAAAIxEQQsAAAAAMJJnmkJpDZpCbdy4cQV+TQBA/rz77rtqPmfOHDV3ajqjcWo49dhjj6l5Wlqa33MDpoqJiVHza6+9toBXAq86e/asmp85c8aWFS9eXB1bqlSpgK65fft2W9ahQ4eA5kBwcIcWAAAAAGAkCloAAAAAgJEoaAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkn2VZll8Dfb5QryUkWrVq5Vf2Z8aOHRuUtSDv/NymIWPq/oc3uL3/RTgDcJfbZ4D9HxpaV/A777wzoDnef/99Nb/tttvytKbCyO39L2LuGbjvvvts2WuvvRbQHE899ZSaz5o1y5b98MMPAc0N//zVGeAOLQAAAADASBS0AAAAAAAjUdACAAAAAIxEQQsAAAAAMBIFLQAAAADASJ7vcgxvcLvDH/sfbnJ7/4twBuAut88A+x9ucnv/i3AG4C66HAMAAAAAPImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEbyWZZlub0IAAAAAAACxR1aAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRwqqgnT17tvh8PklLSwvo/2vVqpXEx8cHdS1Vq1aVPn36BHVO4M+w/xHuOAMIZ+x/hDvOgHeFVUHrZTt37pSOHTtKmTJlJDY2VuLj4+Vf//qX28sCCgT7H+Fq9+7dctddd8mVV14psbGxUq5cOWnZsqWsXLnS7aUBBSIjI0NGjBghcXFxEhMTI02bNpUPPvjA7WUBBeLTTz+VgQMHSr169aR48eJyxRVXSNeuXSU1NdXtpRWoSLcXgPxbu3atdOjQQa655hp58sknpUSJEvLdd9/Jjz/+6PbSgJBj/yOc/fDDD3Lq1Cnp3bu3xMXFydmzZ2Xp0qXSsWNHSU5Oln79+rm9RCCk+vTpI0uWLJEhQ4ZIzZo1Zfbs2dKuXTtJSUmRG264we3lASE1adIk2bx5s9x1111y9dVXy88//yzTpk2Ta6+9Vj7++OOg31kurChoDXfy5Enp1auXtG/fXpYsWSIREdx0R/hg/yPctWvXTtq1a5crGzhwoDRq1EimTJlCQQtP++STT2ThwoUyefJkGTZsmIiI9OrVS+Lj42X48OGyZcsWl1cIhNbQoUNl/vz5UqxYsZysW7duUr9+fXn22WfljTfecHF1BSesX/2tWLFC2rdvL3FxcRIVFSXVq1eXCRMmSFZWljp+x44dkpCQIDExMVKtWjWZPn26bUxGRoaMGTNGatSoIVFRUVK5cmUZPny4ZGRkhORzmD9/vvzyyy8yceJEiYiIkDNnzkh2dnZIrgVvYf8j3HnhDGiKFCkilStXluPHjxfYNWEeL+z/JUuWSJEiRXJ94yY6Olr69u0rW7dulf3794fkuvAGL5yBhISEXMWsiEjNmjWlXr168s0334TkmoVRWN+hnT17tpQoUUKGDh0qJUqUkPXr18vo0aPl5MmTMnny5Fxjjx07Ju3atZOuXbtK9+7dZfHixTJgwAApVqyY3H///SIikp2dLR07dpRNmzZJv379pG7duvLll1/K1KlTJTU1VZYvX+64luzsbDl69Khf6y5VqpQULVpURETWrVsnJUuWlAMHDkinTp0kNTVVihcvLj179pSpU6dKdHR03h4ceB77H+HOC2fgojNnzkh6erqcOHFC3nnnHVm9erV069YtsAcEYcUL+/+zzz6TWrVqScmSJXONadKkiYiI7Nq1SypXruzvQ4Iw44UzoLEsS3755RepV6+eX/N5ghVGZs2aZYmItW/fPsuyLOvs2bO2MUlJSVZsbKx17ty5nCwxMdESEeuFF17IyTIyMqyGDRtaFSpUsDIzMy3Lsqx58+ZZERER1kcffZRrzunTp1siYm3evDknq1KlitW7d++cv+/bt88SEb/+pKSk5Px/V199tRUbG2vFxsZagwYNspYuXWoNGjTIEhHr7rvvzs/DBY9h/yPcefEM/H7dFz8eERFhdenSxTp69GheHiZ4lBf3f7169azWrVvbPo/du3dbImJNnz49oMcI3ubFM6CZN2+eJSLWzJkz/X1ojBfWd2hjYmJy/vvUqVOSkZEhLVq0kOTkZNmzZ480aNAg5+ORkZGSlJSU8/dixYpJUlKSDBgwQHbs2CHNmjWTt956S+rWrSt16tSRI0eO5Ixt3bq1iIikpKRIQkKCupaKFSv63ZXv9+s6ffq0nD17Vvr375/T1bVz586SmZkpycnJMn78eKlZs6Zf8yK8sP8R7rxwBi4aMmSIdOnSRQ4ePCiLFy+WrKwsyczM9Gs+hCcv7P/09HSJioqyjbn40znp6el+zYnw5IUz8Ed79uyRhx9+WJo3by69e/f2az4vCOuCdvfu3TJq1ChZv369nDx5MtfHTpw4kevvcXFxUrx48VxZrVq1REQkLS1NmjVrJnv37pVvvvlGypcvr17v0KFDjmuJjo6WNm3aBPw5XDyM3bt3z5Xfc889kpycLFu3buUFPVTsf4Q7L5yBi+rUqSN16tQRkd+a4rRt21Y6dOgg27ZtE5/Pl+d54V1e2P8xMTHq7yaeO3cu5+OAEy+cgd/7+eefpX379lKqVKmc3y8PF2Fb0B4/flwSExOlZMmSMn78eKlevbpER0fLzp07ZcSIEXlqLJOdnS3169eXKVOmqB//s9/jyMrKksOHD/t1nTJlyuT8AnhcXJzs3r1bLr300lxjKlSoICK//cw/8Efsf4Q7r5wBJ126dJGkpCRJTU2V2rVr+zUvwodX9n+lSpXkwIEDtjE//fSTiPz2HAFovHIGLjpx4oTcdtttcvz4cfnoo4/Cbu+HbUG7YcMG+fXXX2XZsmXSsmXLnHzfvn3q+IMHD8qZM2dyfXfm4psWV61aVUREqlevLp9//rncdNNNAX9HfP/+/VKtWjW/xqakpEirVq1ERKRRo0bywQcfyIEDB3K9aDl48KCIiON3iRDe2P8Id145A04u/qjlH+8yACLe2f8NGzaUlJQUOXnyZK7GUNu2bcv5OKDxyhkQ+e0nEjp06CCpqamybt06ueqqqwK6theEbUF78Ta8ZVk5WWZmprzyyivq+AsXLkhycrIMHTo0Z2xycrKUL19eGjVqJCIiXbt2lVWrVslrr71me++/9PR0yc7Otv24wkV5/dn5rl27yrPPPiszZ87M+Rl9EZHXX39dIiMj//JFD8IT+x/hzitn4NChQzk/kXDR+fPnZe7cuRITExOWL2zw17yy/7t06SLPP/+8zJgxI+d9aDMyMmTWrFnStGlTOhzDkVfOQFZWlnTr1k22bt0qK1askObNm/s1h9eEbUGbkJAgpUuXlt69e8vgwYPF5/PJvHnzcm3s34uLi5NJkyZJWlqa1KpVSxYtWiS7du2SGTNm5LTO7tmzpyxevFj69+8vKSkpcv3110tWVpbs2bNHFi9eLGvWrJHGjRur8+f1Z+evueYauf/+++X//u//5MKFC5KYmCgbNmyQt956S0aOHBl2P3IA/7D/Ee68cgaSkpLk5MmT0rJlS7nsssvk559/ljfffFP27NkjL7zwgpQoUSLgOeF9Xtn/TZs2lbvuuktGjhwphw4dkho1asicOXMkLS1NZs6cGfB8CB9eOQOPPvqovPPOO9KhQwc5evSovPHGG7k+3qNHj4DnNJJb7ZXd8Md23Zs3b7aaNWtmxcTEWHFxcdbw4cOtNWvW2FpiJyYmWvXq1bO2b99uNW/e3IqOjraqVKliTZs2zXaNzMxMa9KkSVa9evWsqKgoq3Tp0lajRo2scePGWSdOnMgZ98d23fmRmZlpjR071qpSpYpVtGhRq0aNGtbUqVODMje8g/2PcOfFM7BgwQKrTZs21qWXXmpFRkZapUuXttq0aWOtWLEi33PDW7y4/y3LstLT061hw4ZZFStWtKKioqzrrrvOev/994MyN7zFi2fg4lsKOf0JFz7LcvhWBAAAAAAAhViE2wsAAAAAACAvKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGCnS34E+ny+U6wD+lNtvl8z+h5vc3v8inAG4y+0zwP6Hm9ze/yKcAbjrr84Ad2gBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaKdHsBsGvbtq2aDxw4UM1vvvlmNb/++utt2c6dO/O+MCCPqlatquYLFiywZU8//bQ6duXKlcFcEgAAADyAO7QAAAAAACNR0AIAAAAAjERBCwAAAAAwEgUtAAAAAMBIFLQAAAAAACPR5dhlbdq0sWVvv/22Ova7775T8/r166v5t99+m/eFAXkQHR2t5vPmzVPzb775xpa99957QV0TAADwtksuuUTNH3vsMVt2++23q2OvueYaNT906JCaJycn27KDBw+qY2fOnKnm58+fV3MEhju0AAAAAAAjUdACAAAAAIxEQQsAAAAAMBIFLQAAAADASBS0AAAAAAAj+SzLsvwa6POFei2edvnll6v5V199Zcs++ugjdez999+v5ocPH877wgzh5zYNGfa/fx588EE1Hzp0qJpfd911tuz06dNBXZMXuL3/RQr/GahRo4aaR0VF2bLq1aurYzt27Kjm9913n9/rOH78uJo/9dRTav7mm2/aMqeOmuHM7TNQ2Pe/E+21x6JFi9SxzZs3D2juTz/91Ja9++676lhtn4uI7N+/35bR9dXO7f0vUnjOgPY1XcT5tXOjRo1CuRy//fjjj2q+YMECW/b666+rY8P53Uv+6gxwhxYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYCS6HAeZUze1adOmqfmuXbts2YABA4K5JE9wu8Mf+99O6yq7fft2dewzzzyj5pMmTQrqmrzK7f0v4s4ZmDBhgi27/vrr1bGNGzdW8+LFi9syp8czIyNDzVeuXKnmt9xyiy0rWbKkOtbpmp9//rktKyxdOQsTt8+Aqc8BX3zxhS2rVauWOjYtLU3NL730UjV32uuBWL9+vS3r27evOvZ///tfvq9nKrf3v0jhOQPFihVT86VLl6p5vXr1bNlLL70U0DXLlSun5v3797dll1xyiTq2aNGifl/P6Sxqzzki4dH9mC7HAAAAAABPoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkWgKFWTz5s1T86uuukrNaf7hH7cbIrD/7ebPn2/LqlSpoo5t2bKlmmdlZQV1TV7l9v4XCe0Z0BqMiYhs2rTJljk153Cyf/9+WzZr1ix17NmzZ9X83XffVfMNGzbYsvLly6tjnf4NU1NTbZnT80U4c/sMmPoc0KlTJ1vm1LRm1apVal6qVCk1T05OtmWtW7f2f3EOfvnlFzW/55571Fw7h17j9v4XMfcMFLT27dur+T//+U811xoaOjW+cjqjnTt3VvPz58+ruYloCgUAAAAA8CQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYCS6HPuhdOnSav6vf/3Llt1xxx3q2DFjxqj51KlT876wMOJ2h79w3v833HCDmv/nP/+xZU7dWb/77rugrincuL3/Rdw5Aw8++KAte/XVV9Wxw4YNU/OVK1faMqf96NTN9YsvvlDzyy67zJY5PU5btmxR8zZt2tiy/6+9u4/Vev7/AP5JuiEhlptlbZZly11ITHMzdy2knXVDNZRKK8PcJYbclZtkjKLEGRm5W82Y3ERDrIaZ1DCZu7AoxdRSp/P74/fbfrbP6/P9Xtc553Jd73Mejz+fe+39eXfO+33O9fKZ19m2bVtY25ZV+w60hd8BPXv2DPNZs2aFeTRZ9c8//wxrV69eHebR1PLevXuHtVu3bg3zK664Isznz58f5imq9vnPsrZxB6ohmtJ90kknlbXGkUceGeZF9y5FphwDAADQKmloAQAASJKGFgAAgCRpaAEAAEiShhYAAIAk7VrtDdSS9u3bh/njjz8e5oMGDcplY8aMCWufe+65Ju8Lqmny5Mlh/uyzz+Yy04xpSfX19bls+fLlYW3R2StnYnCnTp3CPJpmXK7XX389zE005t/WoUOHML/uuuvCPJpmnGVZ9tNPP+Wyyy67LKx9+eWXS9xdlk2cODHMZ8yYEebXX399mK9atSqXrVixouR9wL9h48aN1d5Cq+ANLQAAAEnS0AIAAJAkDS0AAABJ0tACAACQJEOh/mHevHlhPmTIkDC/4YYbcpnhT6Sqrq4uzEeMGBHmZ5xxRiW3A9mOHTty2Zo1ayr2vPXr14f5hAkTwnzOnDm5rGiw1IABA8J8v/32K3kf0BImTZoU5kUDAItEgzE///zzJu3pn+bOnRvmvXr1CvNrrrkmzJcsWZLL+vbtG9Z+9913pW0OWtgvv/xS7S20Ct7QAgAAkCQNLQAAAEnS0AIAAJAkDS0AAABJ0tACAACQpDY55XjBggVhPnr06DC/9957y8ohRZ07dw7zoqmy7733XiW3AzWjvr4+zA877LBcdtVVV4W1Z555ZpgvXbo0l11++eVh7bJlywp2CLFLL700l912221h7c6dO8P8xhtvDPNKThyP3HTTTWG+zz77hPnYsWNzWTT5OMuKp/avW7euxN3Bfxb9vsiyLBs2bFjJa2zatCnMt27d2pQttSre0AIAAJAkDS0AAABJ0tACAACQJA0tAAAASdLQAgAAkKR2jY2NjSUVtmtX6b1URO/evXPZxx9/HNYuXrw4zMeNGxfmf//9d5P3RXlKPKYVk+r5L8cjjzwS5kVTHu+8885Kbod/qPb5z7K2cQdawvjx48O8aCr+nnvumcu2b98e1hZNP54/f36Ju0tXte9Aquf/s88+y2VF01ZXrFgR5ieeeGKL7unfEn2mGzx4cFh7yy23hPn06dNbcktNVu3zn2Xp3oF/W9euXcP80UcfDfMLLrig5LUffvjhML/yyitLXiNV/+0OeEMLAABAkjS0AAAAJElDCwAAQJI0tAAAACRJQwsAAECSWv2U47Vr1+ayHj16hLXHHHNMmK9Zs6ZF99RU0cTmLMuy8847r+Q13nnnnTAvmvxcK6o94S/V81+OTz/9NMxffPHFMG+JKcf9+/fPZTfffHNYe+qpp4Z5Q0NDmA8cODCXFU3xrHXVPv9Z1jbuQCUddNBBYf7EE0/kstNOO62stSdMmBDm9fX1Za1Ty6p9B2r9/Hfv3j3MV65cmct69uwZ1k6dOjXMZ86c2fSNVdGxxx6by6KvR5Zl2fr168P8nHPOyWWffPJJ8zbWBNU+/1lW+3egVjz00ENhPnny5JLXeOGFF8K86Gf9n3/+WfLaqTLlGAAAgFZJQwsAAECSNLQAAAAkSUMLAABAknat9gYq7eCDD85ll112WVhbK8Ofhg8fHuZPPfVUmHfs2LHktVevXh3mxx9/fJhv3bq15LVJ23fffVextfv16xfm0eCDLl26hLVFwzxOPPHEML/44otzWapDoUjfjz/+GOZnn312LpsxY0ZYe/XVV4f5nDlzSt5HaxoUxf878sgjwzwaALVly5awdunSpS26p2qLBh3ecccdYW3RMMK6urpcVo2hUFTXHnvsEeazZs3KZUOHDi1r7Y0bN+ayW2+9NaxtC8OfmsobWgAAAJKkoQUAACBJGloAAACSpKEFAAAgSRpaAAAAkpTclONOnTqF+YIFC8I8mh62cOHCFt1TKYr2PXbs2FxWzsTKLMuyDz/8MMyjaWgDBw4Ma/fcc88wN+W47dhvv/3CvHfv3iWvUXTOZ86cGeZffvllLhs1alRY+9tvv4X59OnTw7xbt25hDrVkx44duWzKlClh7QknnBDmJ510UpjPnz8/l7Vv377kWtLXrl27XPbXX3+Fta1tem9DQ0Mumzt3blg7ZsyYMD/11FNbcEfUuqK/sjBv3rwwP//880teO+pHsizLRo8encu++OKLktflf3lDCwAAQJI0tAAAACRJQwsAAECSNLQAAAAkSUMLAABAkpKbclw0uXTo0KFhfs899+Sy33//vUX3VIphw4aF+ezZs3PZ999/H9ZG/5Ysy7LHHnsszLt27ZrLNmzYULRF2rjFixeH+dSpU8O8Q4cOuaxv375hbVF+3HHH5bKiacZF3n333TAfMmRIWetArRs5cmSYF/3OaGxszGV33HFHWGvKcesUnYEoayt+/vnnMC/6HDVt2rRcduaZZ4a1b775ZtM3xr9q9913D/Oin4MjRowoee1yphlnWZa98cYbJa9NMW9oAQAASJKGFgAAgCRpaAEAAEiShhYAAIAkJTcUqlwzZ86s2NqdOnXKZXPmzAlrhw8fHubRgI4HH3wwrC0aZnXWWWeF+bx583LZ22+/HdYaFsXXX38d5nvttVeYn3vuubmsc+fOYe3KlSvLemY56urqwnznzp3NXhtqybp165q9xm677RbmvXr1CvO1a9c2+5nUln333TfMBw0aFOavvfZaJbdTE7755pswb9++fS674YYbwlpDoWpT9DPviSeeCGuLPqsXiT6Xjxo1Kqx1PirLG1oAAACSpKEFAAAgSRpaAAAAkqShBQAAIEkaWgAAAJKU3JTjokm/n3zySZgfe+yxueytt95qkb2ccsopuWzMmDFh7ebNm8N87ty5uazo39itW7cwj6YZZ1mWbd++PZdNmzYtrN2xY0eY03asWrUqzDdu3Bjm1113XS57/PHHW3RP/9SxY8cwv+iii8I82h+krEePHs1eo0OHDmHevXv3MDflOA3Lli0L8zVr1uSyPn36hLUHHHBAS26p1SqaEk1tOvnkk3NZudOMN23aFOYjR47MZaYZV4c3tAAAACRJQwsAAECSNLQAAAAkSUMLAABAkjS0AAAAJCm5Kcfbtm0L86JJrJWcchxNUf3hhx/C2qOPPjrMo4nGF198cVh77bXXhvn+++8f5hdeeGEuW758eVgLX331VZi/9tprYT569Ohc1rlz57D2119/bfrG/k80TfA/rT1//vxmPxOqoWia8ZIlS5q99pYtW8I8moZLOhoaGsK8sbGx5DUmTJgQ5vX19U3aU0pMeE5f0V8Cef7555u99tSpU8PcROPa4Q0tAAAASdLQAgAAkCQNLQAAAEnS0AIAAJAkDS0AAABJSm7KcZHHHnsszJ9++ulctn79+rB24cKFYd6nT58wP/3003PZOeecE9Z27do1zJcuXZrL+vbtG9a+//77YX7EEUeE+ddffx3mUI7Zs2eH+eDBg3NZ0dnduXNnmN922225rFOnTmHtpEmTwvyWW24J86KJ6LQtBx54YC7bddf4V1/RlPpKiu5M0VTOXr16hfkuu8T/bTq6d7fffntY+8cffxTskJTddddduezJJ58Maw8//PAwr6urC/NFixY1fWM1ZsSIESXXtsTUXFpe0c/BPfbYo9lr9+/fP8w3b96cy2rpfBx66KG57LTTTgtrr7rqqjB/5plnctmtt97arH1Vgje0AAAAJElDCwAAQJI0tAAAACRJQwsAAECS2jU2NjaWVNiuXaX3UhHR0Jmi//F53bp1Yb7bbruFec+ePXPZ6tWrw9q99947zJcvX57LioZTvfLKK2G+Y8eOMG9NSjymFZPq+a+ksWPH5rL7778/rN1rr73CPPq6Fn2vi+5W0VC01qTa5z/Lav8OHHLIIWH+zjvv5LJvv/02rD3jjDPCvJwBY0XDZYoGOkVDzTp06FDy87Ks+HsTnZtoSFaWZdmvv/5a1jP/bdW+A7V+/stx9913h/nVV18d5kWfMWbNmpXLij6nfPTRR2He0NAQ5i0h+txVNBRt4sSJYf7XX3/lsh49eoS1W7duLX1zZar2+c+y2r8DRUOhxo8fn8seeeSRFnlmdH5racBex44dc1mXLl3KWuOBBx7IZddcc01Tt9Rk/+0OeEMLAABAkjS0AAAAJElDCwAAQJI0tAAAACRJQwsAAECSWv2U48jxxx8f5pMmTSprnWhq5dq1a8PaV199NczffvvtXLZhw4ay9tEWVHvCX2s6/5VUNGl23LhxYT5hwoRctmnTprC26N62hftS7fOfZbV/B959990wHzBgQC4r+nred999Yd6tW7cwHzJkSC7r3r17WFvJ72E0iTXLsmzRokW57JJLLglrd+7c2aJ7amnVvgO1fv5bwsiRI8N83rx5Yb777ruXvPbs2bNLXnv79u1h7ZdffhnmdXV1YX7zzTfnsqOOOiqs3bx5c5hPmzYtlz300ENhbSVV+/xnWbp3IJp+HE0+zrKWm37cmphyDAAAABWkoQUAACBJGloAAACSpKEFAAAgSRpaAAAAktQmpxyTnmpP+HP+qaZqn/8sq/07EE2dz7IsW7ZsWS478MADK7aPoq9TS3wP6+vrw/zuu+8O86Kp+ymq9h2o9fNfSf369QvzKVOm5LKhQ4c2+3nbtm0L8w8++CDM+/fvH+ZdunQp+ZkjRowI85deeqnkNSqp2uc/y1rXHSj6t+y///5hXs5fQenRo0eYjx07tuQ1in7Wr1u3ruQ1inz66adh/sorr4R5NAG/oaGh2fsolynHAAAAtEoaWgAAAJKkoQUAACBJGloAAACSZCgUSaj2QATnn2qq9vnPsnTvQJ8+fXLZfffdF9aeddZZzX7epk2bwvzOO+8M8zfeeKPktYuGPBUN0WlNqn0HUj3/lbTLLvl3Ij179gxri4a2nXvuuSU/b/DgwWG+ZcuWMF+6dGkuW7JkSVgbDY/Lstq5W9U+/1nmDlBdhkIBAADQKmloAQAASJKGFgAAgCRpaAEAAEiShhYAAIAkmXJMEqo94c/5p5qqff6zzB2guqp9B5x/qqna5z/L3AGqy5RjAAAAWiUNLQAAAEnS0AIAAJAkDS0AAABJ0tACAACQJA0tAAAASdLQAgAAkCQNLQAAAEnS0AIAAJAkDS0AAABJ0tACAACQJA0tAAAASdLQAgAAkCQNLQAAAEnS0AIAAJAkDS0AAABJ0tACAACQJA0tAAAASdLQAgAAkCQNLQAAAEnS0AIAAJAkDS0AAABJatfY2NhY7U0AAABAubyhBQAAIEkaWgAAAJKkoQUAACBJGloAAACSpKEFAAAgSRpaAAAAkqShBQAAIEkaWgAAAJKkoQUAACBJ/wNW5k2GjWiBHAAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7QAAAPGCAYAAADTLdZkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAACY0UlEQVR4nOzde5yN9f7//9cawxyI7RhTQs4ZUeQwxUhSEckWKadShkK2xFZyTCVF7ewyysepnEKkiGwjOaSQDsoelWmLCjkzZpi5fn/0Nb+m63XVWjNrzTXvaz3ut5vbLc95977es7zfs9ZrrpnX8lmWZQkAAAAAAIaJcHsBAAAAAADkBQUtAAAAAMBIFLQAAAAAACNR0AIAAAAAjERBCwAAAAAwEgUtAAAAAMBIFLQAAAAAACNR0AIAAAAAjERBCwAAAAAwUlgVtLNnzxafzydpaWkB/X+tWrWS+Pj4oK6latWq0qdPn6DOCfwZ9j/CHWcA4Yz9j3DHGfCusCpovejTTz+VgQMHSr169aR48eJyxRVXSNeuXSU1NdXtpQEFIiMjQ0aMGCFxcXESExMjTZs2lQ8++MDtZQGumDhxovh8vqC/+AIKq71798rdd98tl19+ucTGxkqdOnVk/PjxcvbsWbeXBoRcnz59xOfzOf45cOCA20ssEJFuLwD5M2nSJNm8ebPcddddcvXVV8vPP/8s06ZNk2uvvVY+/vhjXtTA8/r06SNLliyRIUOGSM2aNWX27NnSrl07SUlJkRtuuMHt5QEF5scff5Snn35aihcv7vZSgAKxf/9+adKkiZQqVUoGDhwoZcqUka1bt8qYMWNkx44dsmLFCreXCIRUUlKStGnTJldmWZb0799fqlatKpdddplLKytYFLSGGzp0qMyfP1+KFSuWk3Xr1k3q168vzz77rLzxxhsurg4IrU8++UQWLlwokydPlmHDhomISK9evSQ+Pl6GDx8uW7ZscXmFQMEZNmyYNGvWTLKysuTIkSNuLwcIuXnz5snx48dl06ZNUq9ePRER6devn2RnZ8vcuXPl2LFjUrp0aZdXCYRO8+bNpXnz5rmyTZs2ydmzZ+Xee+91aVUFL6x/5HjFihXSvn17iYuLk6ioKKlevbpMmDBBsrKy1PE7duyQhIQEiYmJkWrVqsn06dNtYzIyMmTMmDFSo0YNiYqKksqVK8vw4cMlIyMjJJ9DQkJCrmJWRKRmzZpSr149+eabb0JyTXiDF/b/kiVLpEiRItKvX7+cLDo6Wvr27Stbt26V/fv3h+S68AYvnIGLNm7cKEuWLJEXX3wxpNeBd3hh/588eVJERC699NJceaVKlSQiIsL2+gj4PS+cAc38+fPF5/PJPffcU2DXdFtY36GdPXu2lChRQoYOHSolSpSQ9evXy+jRo+XkyZMyefLkXGOPHTsm7dq1k65du0r37t1l8eLFMmDAAClWrJjcf//9IiKSnZ0tHTt2lE2bNkm/fv2kbt268uWXX8rUqVMlNTVVli9f7riW7OxsOXr0qF/rLlWqlBQtWtTx45ZlyS+//JLz3UpA44X9/9lnn0mtWrWkZMmSucY0adJERER27dollStX9vchQZjxwhkQEcnKypJBgwbJAw88IPXr1w/8gUBY8sL+b9WqlUyaNEn69u0r48aNk7Jly8qWLVvk1VdflcGDB/Pj9/hTXjgDf3T+/HlZvHixJCQkSNWqVf2azxOsMDJr1ixLRKx9+/ZZlmVZZ8+etY1JSkqyYmNjrXPnzuVkiYmJlohYL7zwQk6WkZFhNWzY0KpQoYKVmZlpWZZlzZs3z4qIiLA++uijXHNOnz7dEhFr8+bNOVmVKlWs3r175/x93759loj49SclJeVPP8958+ZZImLNnDnT34cGYcCL+79evXpW69atbZ/H7t27LRGxpk+fHtBjBG/z4hmwLMuaNm2aVapUKevQoUM5661Xr16eHiN4l1f3/4QJE6yYmJhcY5544om8PkzwMK+egd9buXKlJSLWK6+8EshDY7ywvkMbExOT89+nTp2SjIwMadGihSQnJ8uePXukQYMGOR+PjIyUpKSknL8XK1ZMkpKSZMCAAbJjxw5p1qyZvPXWW1K3bl2pU6dOrt9fat26tYiIpKSkSEJCgrqWihUr+t2Z9ffr+qM9e/bIww8/LM2bN5fevXv7NR/Ckxf2f3p6ukRFRdnGREdH53wccOKFM/Drr7/K6NGj5cknn5Ty5cv794kD4o39L/Lb25+0bNlS/v73v0vZsmXlvffek6effloqVqwoAwcO9GtOhCevnIHfmz9/vhQtWlS6du3q11xeEdYF7e7du2XUqFGyfv36nN/DuOjEiRO5/h4XF2f70ZVatWqJiEhaWpo0a9ZM9u7dK998843ji4pDhw45riU6OtrWpSxQP//8s7Rv315KlSqV87uFgBMv7P+YmBj191LOnTuX83HAiRfOwKhRo6RMmTIyaNCggP9fhDcv7P+FCxdKv379JDU1VS6//HIREencubNkZ2fLiBEjpHv37lK2bNmA50V48MIZ+L3Tp0/LihUr5JZbbgm7fR+2Be3x48clMTFRSpYsKePHj5fq1atLdHS07Ny5U0aMGCHZ2dkBz5mdnS3169eXKVOmqB//s9/ly8rKksOHD/t1nTJlytgaHZw4cUJuu+02OX78uHz00UcSFxfn/8IRdryy/ytVqqS+x9pPP/0kIsI5gCMvnIG9e/fKjBkz5MUXX5SDBw/mfPzcuXNy/vx5SUtLk5IlS0qZMmUC+0TgeV7Y/yIir7zyilxzzTU5xexFHTt2lNmzZ8tnn32W7yIB3uSVM/B7y5cvD7vuxheFbUG7YcMG+fXXX2XZsmXSsmXLnHzfvn3q+IMHD8qZM2dyfXcmNTVVRCTnl66rV68un3/+udx0003i8/kCWs/+/fulWrVqfo1NSUmRVq1a5fz93Llz0qFDB0lNTZV169bJVVddFdC1EX68sv8bNmwoKSkpcvLkyVyNobZt25bzcUDjhTNw4MAByc7OlsGDB8vgwYNt46pVqyaPPPIInY9h44X9LyLyyy+/qG/Lc/78eRERuXDhQkDrQPjwyhn4vTfffFNKlCghHTt2DOjaXhC2Be3FH8e1LCsny8zMlFdeeUUdf+HCBUlOTpahQ4fmjE1OTpby5ctLo0aNRESka9eusmrVKnnttddyvY2IyG+/y5edne3YcS+vPzuflZUl3bp1k61bt8qKFSts70UFaLyy/7t06SLPP/+8zJgxI+d9aDMyMmTWrFnStGlTOhzDkRfOQHx8vLz99tu2j48aNUpOnTolL730klSvXt2vORFevLD/RX77kc+1a9dKampqzo9/iogsWLBAIiIi5Oqrr/ZrToQfr5yBiw4fPizr1q2T7t27S2xsrF/zeEnYFrQJCQlSunRp6d27twwePFh8Pp/Mmzcv18b+vbi4OJk0aZKkpaVJrVq1ZNGiRbJr1y6ZMWNGTuvsnj17yuLFi6V///6SkpIi119/vWRlZcmePXtk8eLFsmbNGmncuLE6f15/dv7RRx+Vd955Rzp06CBHjx6VN954I9fHe/ToEfCc8D6v7P+mTZvKXXfdJSNHjpRDhw5JjRo1ZM6cOZKWliYzZ84MeD6EDy+cgXLlykmnTp1s+cU7strHABFv7H8Rkccee0xWr14tLVq0kIEDB0rZsmXl3XffldWrV8sDDzzAr53AkVfOwEWLFi2SCxcuhOWPG4tIeL9tz+bNm61mzZpZMTExVlxcnDV8+HBrzZo1tpbYF98CYfv27Vbz5s2t6Ohoq0qVKta0adNs18jMzLQmTZpk1atXz4qKirJKly5tNWrUyBo3bpx14sSJnHF/bNedVxdbiTv9AS7y4v63LMtKT0+3hg0bZlWsWNGKioqyrrvuOuv9998PytzwFq+egT/ibXug8er+37Ztm3XbbbdZFStWtIoWLWrVqlXLmjhxonX+/PmgzA/v8OoZsCzLatasmVWhQgXrwoULQZvTJD7LcvhWBAAAAAAAhViE2wsAAAAAACAvKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGCnS34E+ny+U6wD+lNtvl8z+h5vc3v8inAG4y+0zwP6Hm9ze/yKcAbjrr84Ad2gBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaKdHsByL9KlSqpeZkyZWzZhQsX1LH//e9/g7omFF7XXnutmvft21fNBwwYoOYrVqywZWvXrs37wv6fr7/+Ws0//PDDfM8NAAAAb+EOLQAAAADASBS0AAAAAAAjUdACAAAAAIxEQQsAAAAAMBIFLQAAAADASD7Lsiy/Bvp8oV4L/kKNGjXUPCUlRc217sfnz59Xx7766qtqPnToUD9XF1p+btOQMXX/N2zY0JatWrVKHXvppZeGeDX+OXbsmJpv3LhRzadMmaLmP/74oy1LS0vL87rc5Pb+FzH3DMAb3D4D7H+4ye39L8IZgLv+6gxwhxYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJplD50LJlS1v21ltvqWOdHuZZs2b5Na+ISHx8vJqXKFEioGtqnJpFbd682Za1adPG73mDxe2GCIV9/2vNn0REli1bZsuqVKkS4tXkj9NjHege+Prrr23Z/Pnz1bHPP/+8mjudi4Lm9v4XKTxnwOnr4Pr169U8OTnZlj355JNBXZPbevTooeZ33XWXLbv//vvVsb/++mtQ1xRsbp+BwrL/w5lT48J77rlHzZ2eFzUvv/yymm/fvt3vOULJ7f0vwhmAu2gKBQAAAADwJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJLoc++Fvf/ubmu/YscOWVa1aVR0bjA51Bw8eVPOhQ4f6PceYMWPUvG7dumq+du1aW9auXTu/rxcsbnf4K+z7//PPP1dzp46whVmwuhwHwqnD5ZAhQ0J2zUC4vf9FCs8ZmDJlipr/4x//UPMvvvjClt1xxx3q2LS0tDyvy027d+9W86uuusqWLVmyRB2rdUQuTNw+A4Vl/3tNkSJFbNnw4cPVsU6vdZz+bcqUKeP3Ov7zn/+o+c033+z3HKHk9v4XKTxnoE6dOmr+4osvqvlll11my5y6VzvN4fQaCwWHLscAAAAAAE+ioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaKdHsBhUmTJk3U/KmnnlLzKlWq5Puas2bNsmXff/+932NFRH7++We/rzdhwgS/x4qIfPfddwGNR/jYs2ePLXPqHpuRkaHm3bt3t2UtWrRQxzp1G09ISHBYof8eeughNde6Oj766KPq2AsXLuR7HchN+ze//PLL8z1HVFRUHlfkLqcu+rGxsX7PcdNNNwVpNYD/GjRooOZjx461ZU7PI3PmzFHzcePGqfn+/ftt2dy5c9WxrVu3VvNAVKxYUc0DeY2Gv3bppZeq+S233OL3HE7vANGjRw81T01NVfNNmzb5fU0nq1atsmXp6enq2M6dO6v5ggUL8r0Opy7/P/zwQ77nLgjcoQUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEbyWZZl+TVQaY7iNVpzAhGRJ5980u85Nm/erOZa8xsRkQMHDvg9dzD88ssval6uXDk11xpijRkzJqhr8oef2zRkCsv+b9++vZq/+eaban7JJZfk+5qHDx9W8+uvv96WhbKJWJkyZdT8xhtvVPMZM2bYMqfGUoGoXr26mjs1VAgGt/e/iDtnoGXLlrbsww8/DGgO7WtYIF/TC5OJEyeq+eOPP+73HMeOHVNzp/NVWLh9BgrLc0Bh16xZMzWfPXu2mmtfT/v376+OdWqMmZ2d7d/iROSyyy5T89WrV6v5fffdZ8ucXgN9/vnnah6Mrzdu73+RwnMGnJr6Oe0Pp9ffyO3UqVNq/sknn9iyNm3ahHo5Nn91BrhDCwAAAAAwEgUtAAAAAMBIFLQAAAAAACNR0AIAAAAAjERBCwAAAAAwUqTbCyhMdu/ereZvvfWWmn/11Ve2TOuo6ZYHHnjAlpUsWVId69Q9bNGiRUFdE/LniiuuUPNgdDN2smDBAjUPZUdjzdGjR9V86dKlal6zZk1b5tQlNhArV65U8w4dOqh5KLsfe53WqTocNGjQQM0feuihfM/9ww8/5HsOwMmjjz6q5rVr11bzO+64w5a98847QV3T7505c0bN4+Li1PzTTz+1ZaNHj1bHTpkyJe8Lg98yMjLU/P7771fz8ePH27JbbrlFHXvy5Ek179Wrl5pXrlxZzUOlUqVKau7U6btEiRJ+z+30OvKzzz7zew43cYcWAAAAAGAkCloAAAAAgJEoaAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkn+XU3vaPA32+UK8FQbZ+/Xpb1rJlS3Xsf/7zHzVv3769Lbtw4UL+FpYHfm7TkCks+z89PV3NixUrFrJrpqamqnndunVDds1giIqKsmUdO3ZUxy5cuDDf19O6YYqINGvWLN9zu73/Rdw5A8ePH7dlpUqVCmgOrfP8k08+mdclFYgmTZqo+bZt2/I9d4sWLdR806ZN+Z47lNw+A4XlOaAwqVq1qi1z6n7/2muvqfmAAQNsWbD+rbV3BXj55ZfVsbfffruaa53W//GPf6hjz507F8DqAuP2/hfhDBQGtWrVUnOn12PLli2zZRER+r3MrKwsNe/bt68tmzNnjtMSQ+avzgB3aAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkCloAAAAAgJEoaAEAAAAARop0ewHIv6ZNm6r5VVdd5fccTh0I3ehoDGda516R0HZArFKlipr36NHDlr3xxhshW0egMjIybJlTN+8tW7aoeUJCgt/Xi46O9nsschs3bpyalyhRwu85nLqrTp8+PU9rAvDnKlasaMucOuF++OGHaq49d0VG6i9NtY7IIiKtW7dW81tvvdWWffvtt+rYLl26qPnbb7+t5oAb9u7dq+bPPvusmmsdjZ1eLz722GNq7kZH47zgDi0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADASTaEMEh8fr+bvvfeemv/tb3+zZRs3blTHrl27Ns/rgrc5NaK67LLLCngl+Xf06FE1P378eMEuBLk4NR4rUqSI33PExsaq+eWXX27LDhw44Pe8AHQNGzb0e+yRI0fUvH///rbs4YcfVsfWq1dPzY8dO6bmkyZNsmUvv/yyOvbXX39Vc6AwadWqlZrfeeedfs8xZcoUNZ86dWpellRocIcWAAAAAGAkCloAAAAAgJEoaAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkz3c51rriderUSR3bsWNHNW/cuLHf14uI0L9HkJ2dreaffvqpX5mISPfu3dW8bNmyaq51bh07dqw69uTJk2qOwmXTpk1qfsMNNxTwSkR8Pl+BXzNUBg4cqOb79u2zZU6f99VXX63mAwYMUPNXX33Vz9V53/PPP6/m2tfk0qVLq2MrVaqk5gsWLLBl3377bQCrK3ilSpUK2dzjx49X81tvvVXNMzMzQ7YWmM3ptYfm3XffVfPISPvL0M8++0wde99996n5woUL1TwjI8PP1QGFywMPPKDmr732WkDzaO/sMHHixDytqbDjDi0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEjGdTnu0qWLmj/00ENqnpiYaMssywromoGMd+pm7DSH1kE5kK7Kf3ZN7THZuHFjQHOjcNE6toqIXH/99fme26m79k8//aTmM2fOzPc1C4srr7xSzbVzG8qvH+Hqq6++UvOEhARbtnz5cnVs7dq11bxatWp+ZeHixhtvVPPp06er+f333x/K5cAAbdu2VfMRI0b4PYdTt+w77rjDlr3//vt+zwuY7vLLL7dljzzySFDmTkpKsmXHjh0LytyFDXdoAQAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkQp1U6g777zTls2dO1cdW6xYMTU/fPiwLXNq0jJr1iw1P3funJovXLjQljn9svX48ePV/MEHH1TzYDh48GDI5ob33HXXXWq+f//+Al5JwRs6dGi+53B6nNatW5fvucPVnj17bNndd9+tjm3Tpo2aT548OahrMt3p06fV3KkpFMJH37591XzGjBlq/u2339qyQ4cOqWMbNWqk5kWLFvVzdYA3LV261JbFx8cHNIfT12+nJopexB1aAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRCkWX4y5duqi51tHYqZuxU4fiUHYR1owePVrNtY7NoXbvvffasq1bt6pjMzMzQ70cwHU1atRQ8+rVq+d77uPHj6u51gkUebdr1y41/+KLL9R82rRptuyFF15Qx6ampqp5cnKymrdo0cKWDRs2TB0biFatWqm50/Ofk5deesmWjRgxQh2bkZER0Nwww6WXXmrLnnvuOXVsu3bt1Nyp+/H8+fNt2RVXXKGOdXqNpp3PTz/9VB37888/qzlgghtuuEHNGzRo4PccW7ZsUfMBAwbkaU1ewh1aAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRfJZlWX4N9PlCtoj169erecuWLW2ZU6e8gQMHqnkwOjdedtllav7EE0/YsqSkJHWs08OsdfN7+umn1bH33Xefmt9xxx1+X/Mf//iHOvbll19W88LCz20aMqHc/4EoUaKEmn/yySdqXrt2bb/nfuONN9S8d+/efs9RmGgdjd999111bM2aNfN9Pa3jp4hIz5498z232/tfpPCcgXDw008/qXnFihXV/MiRI2quPTc4dcks7Nw+A4V9/0dG6m9a8euvv9oyp8+ldevWar59+/a8L+z/6dq1q5ovXLjQljm9K8SKFSvyvQ5Tub3/RQr/GSgsGjdurOabN29Wc617/YIFC9SxDz30kJo7vcuCl/zVGeAOLQAAAADASBS0AAAAAAAjUdACAAAAAIxEQQsAAAAAMBIFLQAAAADASHpbvBC54YYb1DwxMVHN//vf/9qyBx98MN/rqFq1qpq3atVKzR9//HE1r169ui3LzMxUxz7//PNqrnXtc+oouHLlSjXXuhiKiPztb3+zZZ07d1bHzpkzR81Pnjyp5nDH6dOn1fz8+fP5nrtt27ZqPnfuXDUfNGiQLTtx4kS+1+EkOjpazatUqaLmb7/9ti0LRjfjH3/8Uc1feumlfM8N5IXTuTO1ozGcFS1aVM03btyo5to7PTh9rd+1a1ee1/VXypYt6/dYp67dQGETEWG/L+j0mknrZiwism3bNlsWzt2M84o7tAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEgF2hTqiSeeUHPLstR84cKFfs9do0YNNb/pppts2dNPP62OLVWqlN/XExFZs2aNLRs9erQ61qnRUzC0a9dOzZcvX27LWrRooY7997//reY9e/bM87pQcLTmYiIi8fHxfs9RoUIFNb/33nvV/PLLL7dlH3/8sTr2nXfeUfOOHTvaMp/P5/f1RETuueceNQ+V+vXrqzkN1AAEU7ly5WzZhAkT1LFNmzZV84SEBFsWyuZPUVFRau70WkJrpJmamhrUNQGhMmvWLFtWt25ddazTa4Rhw4bZMpo/BY47tAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEgUtAAAAAAAIxVol+O2bduquVOX48TERFu2efNmdaxTN9cSJUrYsnPnzqlj//e//6m5UxdVrXPxhQsX1LGhtG3bNjXfunWrLevQoYM6VuuEKCJy22232bLVq1cHsDoUhPHjx6v5qVOnbNmzzz4blGtq51PLREQeeeQRNY+OjrZlERH699mys7MDWF1gli1bpuZ9+/a1ZdpjCuSV1hlf626L8HPkyBFbFhsbq449evSommtfYyMjA3vp17BhQzWvXLmyLZsyZYrfY0X0567Dhw/7vzigADz88MNq3qtXL7/n+Ne//qXmmzZtytOakBt3aAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkCloAAAAAgJEoaAEAAAAARirQLsezZs1S8z59+qi51jH166+/VsfOnj1bzT/66CNb9uOPP6pjP/74YzU3VefOnW3ZnDlz1LH33nuvmmvdDelyXPg4ddeeOnWqLdM6f4uIjBgxQs2LFi2a94X9P1qnTSdOXc8DpXXK/OCDD9SxgwcPVvOTJ08GZS2Ak0qVKtmyQLvQLl++PEirQWGnfU0XcX6nh/Xr14dsLVrn+Q8//FAde/vtt6v57t27g7omID9iYmLU3Kl7t2bt2rVqPnny5DytCf7hDi0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADCSz/KzA4vP58v3xaKiotS8evXqfs/h1NCJ5i3+KV++fED5d999Z8syMjKCuiZ/BKtRUF4FY/8Xdj169FDzypUrq/lTTz0VknVEROjfZ0tNTVVzpyYpn332mS3btm1b3hfmIrf3v0h4nAE3vPrqq7asf//+Ac3h1BDISw133D4DhX3/V6xYUc1vuummfM/9ww8/qPmePXts2ZEjR/J9Pdi5vf9FCv8ZCIaJEyeq+eOPP67m3377rS27+uqr1bHp6el5Xxj+8gxwhxYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYKQC7XIM5JXbHf7Y/3CT2/tfhDMQKnQ59o/bZ4D9Dze5vf9FvHUGypYtq+ZpaWlqXqJECTW/5ZZbbNnatWvzvC44o8sxAAAAAMCTKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRIt1eAAAA4UrrcnzttdeqYydOnKjm//vf/4K6JgDwsg4dOqi5UzdjJx999FEwloMg4A4tAAAAAMBIFLQAAAAAACNR0AIAAAAAjERBCwAAAAAwEgUtAAAAAMBIdDkGAMAlX3zxhS1r2rSpCysBgPAQaDdjJ8OGDbNlEyZMCMrcCAx3aAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkCloAAAAAgJF8lmVZfg30+UK9FsCRn9s0ZNj/cJPb+1+EMwB3uX0G2P9wk9v7X4QzAHf91RngDi0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEh+dzkGAAAAAKAw4Q4tAAAAAMBIFLQAAAAAACNR0AIAAAAAjERBCwAAAAAwEgUtAAAAAMBIFLQAAAAAACNR0AIAAAAAjERBCwAAAAAwEgUtAAAAAMBIYVXQzp49W3w+n6SlpQX0/7Vq1Uri4+ODupaqVatKnz59gjon8GfY/wh3nAGEM/Y/wh1nwLvCqqD1sp07d0rHjh2lTJkyEhsbK/Hx8fKvf/3L7WUBIZeRkSEjRoyQuLg4iYmJkaZNm8oHH3zg9rKAAsVzAMLR7t275a677pIrr7xSYmNjpVy5ctKyZUtZuXKl20sDCsTp06dlzJgxcuutt0qZMmXE5/PJ7Nmz3V5WgYt0ewHIv7Vr10qHDh3kmmuukSeffFJKlCgh3333nfz4449uLw0IuT59+siSJUtkyJAhUrNmTZk9e7a0a9dOUlJS5IYbbnB7eUDI8RyAcPXDDz/IqVOnpHfv3hIXFydnz56VpUuXSseOHSU5OVn69evn9hKBkDpy5IiMHz9errjiCmnQoIFs2LDB7SW5goLWcCdPnpRevXpJ+/btZcmSJRIRwU13hI9PPvlEFi5cKJMnT5Zhw4aJiEivXr0kPj5ehg8fLlu2bHF5hUBo8RyAcNauXTtp165drmzgwIHSqFEjmTJlCgUtPK9SpUry008/ScWKFWX79u1y3XXXub0kV4T1M9+KFSukffv2EhcXJ1FRUVK9enWZMGGCZGVlqeN37NghCQkJEhMTI9WqVZPp06fbxmRkZMiYMWOkRo0aEhUVJZUrV5bhw4dLRkZGSD6H+fPnyy+//CITJ06UiIgIOXPmjGRnZ4fkWvAWL+z/JUuWSJEiRXK9aImOjpa+ffvK1q1bZf/+/SG5LrzBC2eA5wDklRf2v6ZIkSJSuXJlOX78eIFdE2bywhmIioqSihUrhmRuk4T1HdrZs2dLiRIlZOjQoVKiRAlZv369jB49Wk6ePCmTJ0/ONfbYsWPSrl076dq1q3Tv3l0WL14sAwYMkGLFisn9998vIiLZ2dnSsWNH2bRpk/Tr10/q1q0rX375pUydOlVSU1Nl+fLljmvJzs6Wo0eP+rXuUqVKSdGiRUVEZN26dVKyZEk5cOCAdOrUSVJTU6V48eLSs2dPmTp1qkRHR+ftwYHneWH/f/bZZ1KrVi0pWbJkrjFNmjQREZFdu3ZJ5cqV/X1IEGa8cAZ4DkBeeWH/X3TmzBlJT0+XEydOyDvvvCOrV6+Wbt26BfaAIOx46QyEPSuMzJo1yxIRa9++fZZlWdbZs2dtY5KSkqzY2Fjr3LlzOVliYqIlItYLL7yQk2VkZFgNGza0KlSoYGVmZlqWZVnz5s2zIiIirI8++ijXnNOnT7dExNq8eXNOVqVKFat37945f9+3b58lIn79SUlJyfn/rr76ais2NtaKjY21Bg0aZC1dutQaNGiQJSLW3XffnZ+HCx7jxf1fr149q3Xr1rbPY/fu3ZaIWNOnTw/oMYK3efEM8BwAf3lx//9+3Rc/HhERYXXp0sU6evRoXh4meJiXz4BlWdann35qiYg1a9asAB8Z84X1HdqYmJic/z516pRkZGRIixYtJDk5Wfbs2SMNGjTI+XhkZKQkJSXl/L1YsWKSlJQkAwYMkB07dkizZs3krbfekrp160qdOnXkyJEjOWNbt24tIiIpKSmSkJCgrqVixYp+d2b9/bpOnz4tZ8+elf79++d0tOzcubNkZmZKcnKyjB8/XmrWrOnXvAgvXtj/6enpEhUVZRtz8a5Uenq6X3MiPHnhDPAcgLzywv6/aMiQIdKlSxc5ePCgLF68WLKysiQzM9Ov+RC+vHQGwl1YF7S7d++WUaNGyfr16+XkyZO5PnbixIlcf4+Li5PixYvnymrVqiUiImlpadKsWTPZu3evfPPNN1K+fHn1eocOHXJcS3R0tLRp0ybgz+HiYezevXuu/J577pHk5GTZunUrL2ag8sr+134v5dy5czkfB5x45QyI8ByAwHlh/19Up04dqVOnjoj81hiwbdu20qFDB9m2bZv4fL48zwtv89IZCHdhW9AeP35cEhMTpWTJkjJ+/HipXr26REdHy86dO2XEiBF5aqqRnZ0t9evXlylTpqgf/7Pf5cvKypLDhw/7dZ0yZcpIsWLFROS3A7Z792659NJLc42pUKGCiPz2M//AH3ll/1eqVEkOHDhgG/PTTz+JyG/nA9B45QzwHIC88Mr+d9KlSxdJSkqS1NRUqV27tl/zIrx4/QyEm7AtaDds2CC//vqrLFu2TFq2bJmT79u3Tx1/8OBBOXPmTK7vzqSmpoqISNWqVUVEpHr16vL555/LTTfdFPB3BPfv3y/VqlXza2xKSoq0atVKREQaNWokH3zwgRw4cCDXF+2DBw+KiDh+lwjhzSv7v2HDhpKSkiInT57M1Rhq27ZtOR8HNF45AzwHIC+8sv+dXPx1kz/eZQMu8voZCDdhW9AWKVJEREQsy8rJMjMz5ZVXXlHHX7hwQZKTk2Xo0KE5Y5OTk6V8+fLSqFEjERHp2rWrrFq1Sl577TXbe5+lp6dLdna27ccVLsrrz8537dpVnn32WZk5c2bOz+iLiLz++usSGRnJhofKK/u/S5cu8vzzz8uMGTNy3oc2IyNDZs2aJU2bNqXDMRx55QzwHIC88Mr+P3ToUM5PI1x0/vx5mTt3rsTExMhVV13l15wIP145A/hN2Ba0CQkJUrp0aendu7cMHjxYfD6fzJs3L9fG/r24uDiZNGmSpKWlSa1atWTRokWya9cumTFjRk7r7J49e8rixYulf//+kpKSItdff71kZWXJnj17ZPHixbJmzRpp3LixOn9ef3b+mmuukfvvv1/+7//+Ty5cuCCJiYmyYcMGeeutt2TkyJH8yCVUXtn/TZs2lbvuuktGjhwphw4dkho1asicOXMkLS1NZs6cGfB8CB9eOQM8ByAvvLL/k5KS5OTJk9KyZUu57LLL5Oeff5Y333xT9uzZIy+88IKUKFEi4DkRHrxyBkREpk2bJsePH8/5yZyVK1fKjz/+KCIigwYNklKlSuVpXqO41V7ZDX9s171582arWbNmVkxMjBUXF2cNHz7cWrNmja0ldmJiolWvXj1r+/btVvPmza3o6GirSpUq1rRp02zXyMzMtCZNmmTVq1fPioqKskqXLm01atTIGjdunHXixImccX9s150fmZmZ1tixY60qVapYRYsWtWrUqGFNnTo1KHPDO7y6/9PT061hw4ZZFStWtKKioqzrrrvOev/994MyN7zFq2eA5wD4w4v7f8GCBVabNm2sSy+91IqMjLRKly5ttWnTxlqxYkW+54b3ePEMXJxLHN7i5+Ln6nU+y3L4VgQAAAAAAIVYhNsLAAAAAAAgLyhoAQAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABgp0t+BPp8vlOsA/pTbb5fM/oeb3N7/IpwBuMvtM8D+h5vc3v8inAG466/OAHdoAQAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYKRItxcA/918881q/vDDD6t5x44dbdlzzz2njv3nP/+Z94UBAAAAgAu4QwsAAAAAMBIFLQAAAADASBS0AAAAAAAjUdACAAAAAIxEQQsAAAAAMJLPsizLr4E+X6jXEpYqVapky2655RZ17JQpU9S8VKlSfl/v/Pnzau7UKXnmzJl+zx1Kfm7TkGH/w01u738RzkBBKlGihJq/9tpran733Xer+ccff2zLnJ5fTp486efq3OH2GWD/+6dYsWJqHhUV5fccbdq0UfMxY8aoef369f2e22mOp556yu853OD2/hfhDBQkp3/vcePGqfnYsWNDuJrC4a/OAHdoAQAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkShoAQAAAABGinR7AV7j1J2yR48ean7//ffbskaNGgV1Tb9XpEgRNb/kkktCdk0ULpGR+rF/4IEH1LxmzZp+z3369Gk1f/3119X80KFDtiwjI8Pv6wGmq1Onji1btWqVOrZq1apq7tT9sWnTprasZ8+e6th///vfDitEYeL0HF67dm01T0pKCuVybK6++mo1b9GihZprnXMD7egbyHjtTABuCqRDcWJiYugWYjju0AIAAAAAjERBCwAAAAAwEgUtAAAAAMBIFLQAAAAAACPRFCrInJp5XH/99WoeSEMEp2Y5U6dOVfOHH37Ylh07dkwd++KLL6o5vGfUqFEB5YHQ9rOIyBNPPKHmKSkptmzdunXqWKd8x44dfq4OcE+lSpXUfM2aNbascuXK6tgZM2ao+fjx49X822+/tWVOTeFghgoVKqj5F198UcArKfzS09Nt2bJly1xYCRAcrVq1cnsJhRZ3aAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkCloAAAAAgJEoaAEAAAAARqLdoR/q1Kmj5itWrLBlTt0pA3H06FE1f/DBB9V8+fLlaq511VywYEGe1wXzdO/e3ZY9+eST6lin7tqhdOONN/qViYiMHTtWzXfu3KnmixYtsmUffvihOvbzzz93WCEQmJiYGDV36kavPWe8//776thHH31Uzc+cOaPm7777ri376quv1LGA12jPdbNmzXJhJYCzxMREt5fgCdyhBQAAAAAYiYIWAAAAAGAkCloAAAAAgJEoaAEAAAAARqKgBQAAAAAYyWf52drU5/OFei2ui4zUmz6/9NJLat6/f/98X3P//v227B//+Ic69u2338739UzlRgfe3zN1/+/evduWOXXtDsZj7PQ4FZa5T58+reZO3b8HDBjg99yh5Pb+FzH3DBQ0pz3z73//W8337dtnyxo0aKCOddq/TqpWrWrLDhw4oI49f/58QHMXNLfPQGHZ/1FRUWo+bdo0Nb/vvvvyfc3PPvtMzbXnEqcu3060x9Xp3zo9PV3NnTr3v/nmm7bs8OHDAayu8HB7/4sUnjNgqlatWql5SkqK33OMGzdOzZ3eBcJL/uoMcIcWAAAAAGAkCloAAAAAgJEoaAEAAAAARqKgBQAAAAAYSe+C5HFOTXEGDRqk5sFo/uREa9oBBOrll19Wc22vR0To38fKzs7O9zqc5jh48KCaL1y40JatWrVKHfvhhx+qeVxcnJp369bNljk1XEtKSlLz22+/3Zbdeeed6thdu3ap+YULF9Qc5mvcuLEte/HFF9WxR48eVfOuXbvaskCbPzlJS0sLyjwoPDIyMtR88ODBaj5nzpx8X9Ppa9uOHTtsWfXq1fN9vUA/x1mzZuX7mkCoOTWFQnBwhxYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYKSw7HLs1IUvGN2MP/jgAzV36kILBOKSSy5R85YtW6q5ZVm2zKkT8alTp9TcqUvmtddea8vWrl2rjp0wYYKaB4NTB+WpU6fasp9++kkd++abb6p5pUqVbNnHH3+sjn344YfVPDk5Wc1hPq3ratGiRdWxW7duVXOtUywQqPT0dDXftGlTvud26i5cuXLlfM+tdYF/6KGH1LHB6NgMmGzs2LFuL6HQ4g4tAAAAAMBIFLQAAAAAACNR0AIAAAAAjERBCwAAAAAwEgUtAAAAAMBIPktrg6oN9PlCvZYCs379ejVPTEwMaJ7jx4/bsptuukkdu2vXroDmRm5+btOQKSz7v3fv3mo+c+ZMv+dw+lyGDBmi5uHQodupy3G3bt38nuO9995T8zvuuCNPa/o9t/e/SOE5A25o0qSJmm/ZssWWfffdd+rYxo0bq7lTd3Hk5vYZCIf9P2jQIDWfNGmSmhcrVizf1+zTp48te+ONN/I9r9e4vf9FwuMMhFIg/4YbNmxQ8xtvvDFIqzHPXz1+3KEFAAAAABiJghYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABgp0u0FuOHKK68Myjy9evWyZXQzRrDExcXZsmnTpuV73oMHD6r566+/nu+5TfXzzz/ne45KlSoFYSVwk1PX1tmzZ6t5RIT9e8Lz5s1Txzp1M46OjvZ7HSdPnlRzIBAPP/ywmj/33HNqXrRo0ZCthY7G8JqxY8fme45w7macV9yhBQAAAAAYiYIWAAAAAGAkCloAAAAAgJEoaAEAAAAARvJ8U6iRI0fasiuuuCIoc3/00Uf5niM+Pt6WtWjRIqA5brnlFjXv2LGj33OsWLFCzbt162bLMjMz/Z4Xede6dWtbFhsbm+95nZrTpKen53tuU11yySVq7vP5/J5j48aNwVoOXNK5c2c1r1Onjt9z1KpVS8337dun5pGR9qfhIkWKqGPPnTun5gsXLlTzMWPG2LLz58+rY+FNd955py0bOHCgOjaUzZ+caK/RAvX222+r+Z49e/I9NxAo7esuQo87tAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEgUtAAAAAAAI3mmy7HWKVJE72hsWVZAc7/44otqfubMGVvWoEEDdaxTF9VFixbZsooVK/q/uD8RyOfp1BE5OjraltHluGBcc801tizQvat57bXX8j2HqW6//XY179u3r5oH8ngH498G7mrcuHG+5+jRo4eaO33d1M6jUzfj3r17q/k///lPNX///fdtGd24valGjRpqvmTJkgJeSWCefvppW5adnR3QHE899ZSaL1682JY9+eST6thvv/02oGsCKFy4QwsAAAAAMBIFLQAAAADASBS0AAAAAAAjUdACAAAAAIxEQQsAAAAAMJJnuhwXL15czfv165fvuU+ePKnmrVu3tmVvvPGGOrZcuXJq7vP5bFmg3VIzMjLUvGjRorYsIoLvYYS7hQsXur0E17Rr1y5kc9Ml0xyxsbFq3r59+3zP/cMPP6j5448/ruYLFizwe+6lS5eq+ZYtW9Q8OTnZljVq1Egde/bsWb/XAXMU9u7rWkfjYK35rrvusmVNmjRRx3bu3FnNd+/ebcsuXLiQv4XBM8aOHZvvOcaNG5f/hYA7tAAAAAAAM1HQAgAAAACMREELAAAAADASBS0AAAAAwEg+y8/fvteaFxUmpUqVUvOjR48W8EoCE0hTqHfeeUfNp0+fruZaQ5DKlSsHsDqR0qVL2zKnJlmh5HZjCzf2f0pKii1r0aJFQHPs3LnTljk1xfCa0aNH27InnnhCHRsZqffH0/ZdamqqOrZ58+ZqfuLECacl+s3t/S9S+J8DAtGtWzc1D6RBk4jIgQMHbNmNN96ojg1G0zCt0Z+Ic2NATcWKFdX80KFDeVpTQXH7DBT2/V+yZEk1HzBggC2777771LFOzdKc5o6KirJlZ86cUcceOXJEzbXH1amJptPrvFBq1qyZLdu+fXuBr8Pt/S9S+M9AKLVq1UrNtddpgQrnxzUQf3UGuEMLAAAAADASBS0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADCS3toTBea9996zZf/+97/VsZdccomat2/fXs3j4uL8XseePXvU/MKFC37PgeBKTEy0ZYF2Oty4cWOwllNoxcfHq3lSUpItc+pm7NRlMDMz05b16NFDHRuMbsYoGJUqVQrKPKtXr7ZlwehmDATK6d0HJk2a5Fcm4twBu2rVqmr+t7/9zZb9/PPP6thdu3apuaZhw4Zqft1116n5kCFD1Lx27dp+X9PJ448/bsucuqSfP38+39dD4eTU5TgQ48aNy/9C4Ig7tAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEgUtAAAAAAAI9HlOMicutxNmTJFzZ955hlb1rZtW3XswoUL876w/+e///2vmnfs2FHNz549m+9rIm+0jsaBdjkOdHxh5tTNWOsULiJy6aWX2jKnx0PrZiyid8/cuXOnwwoRbpYsWVKg13PqQutk9+7dtuzUqVPBWg48xqlDsVMeKk4dkZ1yp+eADRs22LIrr7wyoLVor43KlCmjjv3ll18Cmhvm0N51IlDafkTwcIcWAAAAAGAkCloAAAAAgJEoaAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkuhwH2fHjxwMa/9Zbb9mym2++OUirsXv00UfV/LvvvgvZNZE3e/futWU1atRwYSUFa/To0WqelJSk5lo340ANGjRIzV9//fV8z43C59dffw3KPOvXrw/KPH8UGak/Nc+ZMyegeebNm2fL0tPT87QmFA5RUVFq3rlzZzXv37+/Lfvf//6njn3ppZfUfPv27X6uLrSuvvpqNX/sscfUPNCOxpoff/zRljl1xYf5WrVqFVAeCLochxZ3aAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkCloAAAAAgJE80xTK5/O5vQQRESlfvryajxgxQs0jIuzfU8jOzg7omrt371bz+fPn27IPPvggoLnhnvfee8+WPfLIIy6sJP9uv/12NR81apQtu+aaa9SxTo1yLMvyex0PPfSQmtP8KbysXbs2KPOULFnSlh09ejSgOYoWLWrLnBr8ODUmOXDggJo7NfmBuYYNG6bm48aN83uO66+/Xs2dvk5///33av7FF1/YslWrVvm9DhGRkSNH2jKnr+mVK1dW8zJlygR0zUDcc889tuzYsWMhux7cFYzmT3AHd2gBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEbyTJfj06dPq3nLli1tmVPnR6fuqqEUSIfW1NRUNe/QoYOa//DDD3laEwoHbU8H2s1b68IaqNjYWDUvW7asLXvyySfVsX379s33Opw+98zMTDUfNGiQLaObMUScOxF/+OGHap6YmKjmWsfZxx9/XB2rdTMW0TsaL1iwQB3r9DzXvn17Nc/IyFBzmKtChQohm/uSSy5R8wYNGvid9+zZM6Bral/XA3ldFKgff/xRzadNm6bmn376acjWgsLH6Wt9MIwdOzagHIHhDi0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEg+y892coF2Vy3MqlSpouYrV65U83r16oVsLR999JEtW7hwoTp23bp1av7tt98GdU2FUSi7HvrDjf2vdbP86quv1LFlypTxe96lS5cGtI7LL79czZs2bWrLnB6nYPz7Oe3/SZMmqXlKSkq+r1lYuL3/Rbz1HOBE64ovIrJ69Wo1T09Pt2VOZ7R48eJq3qhRI1vm1M24Y8eOar5hwwY19xK3z0Bh2f9O79Lw8MMPF/BKgiOUXY7feecdWzZ69Gh1rNO5LSzc3v8ihecMhFIoH+cbb7xRzcPh63cw/NW/DXdoAQAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkcKyKRTM43ZDhMKy/2vVqqXmAwYMUPMHHnjAlsXGxqpjg/EYB9oUav369bbMqfnTc889l/eFGc7t/S9SeM6AG+Li4tR87ty5tqx169bq2OPHj6v5W2+9ZctefvlldWxhb1wTSm6fgcKy/6OiotQ8MjLS7zm6du2q5ldeeWVAa+nfv78tK126dEBzbNy40ZZt3rxZHet0hqZPn67mGRkZtuzChQv+L64QcXv/ixSeMxBKoXwdhPyhKRQAAAAAwJMoaAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkCloAAAAAgJHocgwjuN3hz9T9X6lSJVvm1IW1YcOG+b7emTNn1Pz1119X80OHDtmyzMzMfK/Da9ze/yLmngF4g9tngP0PN7m9/0U4A3AXXY4BAAAAAJ5EQQsAAAAAMBIFLQAAAADASBS0AAAAAAAjUdACAAAAAIxEl2MYwe0Of+x/uMnt/S/CGYC73D4D7H+4ye39L8IZgLvocgwAAAAA8CQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABjJZ1mW5fYiAAAAAAAIFHdoAQAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkShoAQAAAABGCquCdvbs2eLz+SQtLS2g/69Vq1YSHx8f1LVUrVpV+vTpE9Q5gT/D/ke44wwgnLH/Ee44A94VVgWtV+3du1fuvvtuufzyyyU2Nlbq1Kkj48ePl7Nnz7q9NCDkMjIyZMSIERIXFycxMTHStGlT+eCDD9xeFlAg+vTpIz6fz/HPgQMH3F4iEFI7duyQW2+9VUqWLCmXXHKJtG3bVnbt2uX2soACQx0gEun2ApA/+/fvlyZNmkipUqVk4MCBUqZMGdm6dauMGTNGduzYIStWrHB7iUBI9enTR5YsWSJDhgyRmjVryuzZs6Vdu3aSkpIiN9xwg9vLA0IqKSlJ2rRpkyuzLEv69+8vVatWlcsuu8yllQGht3PnTrnhhhukcuXKMmbMGMnOzpZXXnlFEhMT5ZNPPpHatWu7vUQgpKgDfkNBa7h58+bJ8ePHZdOmTVKvXj0REenXr59kZ2fL3Llz5dixY1K6dGmXVwmExieffCILFy6UyZMny7Bhw0REpFevXhIfHy/Dhw+XLVu2uLxCILSaN28uzZs3z5Vt2rRJzp49K/fee69LqwIKxpNPPikxMTGydetWKVu2rIiI9OjRQ2rVqiWPP/64LF261OUVAqFFHfCbsP6R4xUrVkj79u0lLi5OoqKipHr16jJhwgTJyspSx+/YsUMSEhIkJiZGqlWrJtOnT7eNycjIkDFjxkiNGjUkKipKKleuLMOHD5eMjIyQfA4nT54UEZFLL700V16pUiWJiIiQYsWKheS6MJ8X9v+SJUukSJEi0q9fv5wsOjpa+vbtK1u3bpX9+/eH5LrwBi+cAc38+fPF5/PJPffcU2DXhHm8sP8/+ugjadOmTU4xK/Lb65/ExER599135fTp0yG5LrzBC2eAOuA3YX2Hdvbs2VKiRAkZOnSolChRQtavXy+jR4+WkydPyuTJk3ONPXbsmLRr1066du0q3bt3l8WLF8uAAQOkWLFicv/994uISHZ2tnTs2FE2bdok/fr1k7p168qXX34pU6dOldTUVFm+fLnjWrKzs+Xo0aN+rbtUqVJStGhREfntF9UnTZokffv2lXHjxknZsmVly5Yt8uqrr8rgwYOlePHieXtw4Hle2P+fffaZ1KpVS0qWLJlrTJMmTUREZNeuXVK5cmV/HxKEGS+cgT86f/68LF68WBISEqRq1ap+zYfw5IX9n5GRITExMbYxsbGxkpmZKV999ZU0a9bMz0cE4cYLZ4A64P+xwsisWbMsEbH27dtnWZZlnT171jYmKSnJio2Ntc6dO5eTJSYmWiJivfDCCzlZRkaG1bBhQ6tChQpWZmamZVmWNW/ePCsiIsL66KOPcs05ffp0S0SszZs352RVqlSxevfunfP3ffv2WSLi15+UlJRc80+YMMGKiYnJNeaJJ57I68MEj/Li/q9Xr57VunVr2+exe/duS0Ss6dOnB/QYwdu8eAb+aOXKlZaIWK+88kogDw3CgBf3f/369a1atWpZFy5cyLW2K664whIRa8mSJXl6rOBNXjwDlkUdYFmWFdZ3aH//Xb1Tp05JRkaGtGjRQpKTk2XPnj3SoEGDnI9HRkZKUlJSzt+LFSsmSUlJMmDAANmxY4c0a9ZM3nrrLalbt67UqVNHjhw5kjO2devWIiKSkpIiCQkJ6loqVqzod2fW369L5LfW3y1btpS///3vUrZsWXnvvffk6aeflooVK8rAgQP9mhPhxwv7Pz09XaKiomxjoqOjcz4OOPHCGfij+fPnS9GiRaVr165+zYXw5YX9/9BDD8mAAQOkb9++Mnz4cMnOzpannnpKfvrpJxHhOQB/zgtnQIQ6QCTMf+R49+7dMmrUKFm/fn3Oz6BfdOLEiVx/j4uLs922r1WrloiIpKWlSbNmzWTv3r3yzTffSPny5dXrHTp0yHEt0dHRtk6V/li4cKH069dPUlNT5fLLLxcRkc6dO0t2draMGDFCunfvnut3S4CLvLD/Y2Ji1N9LOXfuXM7HASdeOAO/d/r0aVmxYoXccsstfN3HX/LC/u/fv7/s379fJk+eLHPmzBERkcaNG8vw4cNl4sSJUqJEiYDnRPjwwhmgDvhN2Ba0x48fl8TERClZsqSMHz9eqlevLtHR0bJz504ZMWKEZGdnBzxndna21K9fX6ZMmaJ+/M9+ly8rK0sOHz7s13XKlCmT80ver7zyilxzzTU5m/iijh07yuzZs+Wzzz7L94skeI9X9n+lSpXU99m8+N35uLg4v+ZE+PHKGfi95cuX090YfvHS/p84caIMGzZMdu/eLaVKlZL69evL448/LiL/f8EB/JFXzgB1wG/CtqDdsGGD/Prrr7Js2TJp2bJlTr5v3z51/MGDB+XMmTO5vjuTmpoqIpLTeKN69ery+eefy0033SQ+ny+g9ezfv1+qVavm19iUlBRp1aqViIj88ssvajvu8+fPi4jIhQsXAloHwoNX9n/Dhg0lJSVFTp48masx1LZt23I+Dmi8cgZ+780335QSJUpIx44dA7o2wo/X9n/p0qVzve/4unXr5PLLL5c6deoEtA6ED6+cAeqA34RtQVukSBER+e0N6C/KzMyUV155RR1/4cIFSU5OlqFDh+aMTU5OlvLly0ujRo1ERKRr166yatUqee2113K9jYjIb7/HkZ2d7dhtLK8/O1+rVi1Zu3atpKam5vpO5IIFCyQiIkKuvvpqv+ZEePHK/u/SpYs8//zzMmPGjJz3oc3IyJBZs2ZJ06ZN6XAMR145AxcdPnxY1q1bJ927d5fY2Fi/5kH48tr+/71FixbJp59+Ks8//7xERIT1u1PiT3jlDFAH/CZsC9qEhAQpXbq09O7dWwYPHiw+n0/mzZuXa2P/XlxcnEyaNEnS0tKkVq1asmjRItm1a5fMmDEjp3V2z549ZfHixdK/f39JSUmR66+/XrKysmTPnj2yePFiWbNmjTRu3FidP68/O//YY4/J6tWrpUWLFjJw4EApW7asvPvuu7J69Wp54IEH+JFLqLyy/5s2bSp33XWXjBw5Ug4dOiQ1atSQOXPmSFpamsycOTPg+RA+vHIGLlq0aJFcuHCBHzeGX7yy/zdu3Cjjx4+Xtm3bStmyZeXjjz+WWbNmya233iqPPPJIwPMhfHjlDFAH/D9utVd2wx/bdW/evNlq1qyZFRMTY8XFxVnDhw+31qxZY2uJnZiYaNWrV8/avn271bx5cys6OtqqUqWKNW3aNNs1MjMzrUmTJln16tWzoqKirNKlS1uNGjWyxo0bZ504cSJn3B/bdefHtm3brNtuu82qWLGiVbRoUatWrVrWxIkTrfPnzwdlfniDV/d/enq6NWzYMKtixYpWVFSUdd1111nvv/9+UOaGt3j1DFiWZTVr1syqUKFCrrcvAX7Pi/v/22+/tdq2bWuVK1fOioqKsurUqWM988wzVkZGRr7nhvd48QxYFnWAZVmWz7IcvhUBAAAAAEAhxi8XAAAAAACMREELAAAAADASBS0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEiR/g70+XyhXAfwp9x+u2T2P9zk9v4X4QzAXW6fAfY/3OT2/hfhDMBdf3UGuEMLAAAAADASBS0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEgUtAAAAAAAI1HQAgAAAACMFOn2AsLdP/7xD1v2wgsvqGPvu+8+NZ8zZ05Q1wQAAAAAJuAOLQAAAADASBS0AAAAAAAjUdACAAAAAIxEQQsAAAAAMBIFLQAAAADASHQ5LiCrV69W85tuusmWbdiwQR27ZMmSYC4JKDBOHbpHjRply6pVq6aO9fl8am5Zlppr5+Xpp59Wx+7atUvNAQAAULhxhxYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYCSf5dQi9I8DHTqMhrOyZcvaspUrV6pjmzRpoubHjh2zZTfccIM69r///W8Aq/MWP7dpyLD/7SZNmmTLHnnkEXVsZKTeUL2gH1ftvImItGnTRs0LS/djt/e/CGcA7nL7DLD/4Sa3978IZ6AgXXLJJWo+YMCAgOYZP368LYuKilLHjhgxQs2fe+65gK4ZKn91BrhDCwAAAAAwEgUtAAAAAMBIFLQAAAAAACNR0AIAAAAAjERTKD84NWl6+eWXbVmDBg3UsXPmzFHzwYMH27JTp04FsLrw4HZDhHDe/zt27FDzq6++2pZFRJj5PbJ58+apeZ8+fQp2IQ7c3v8i4X0G4D63zwD73z+NGjVS806dOql5+fLlbdmdd97p91gRkW+++UbNly1bZsueeeYZdezZs2fVvLBwe/+LcAby64477lDz4cOH27LatWurY0uXLh3UNf3e+fPn1fzFF1+0Za+//ro69ttvvw3mknKhKRQAAAAAwJMoaAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkCloAAAAAgJHCssuxUyfWZ599Vs0HDhyo5pGRkbbs0UcfVcdOmzZNzQtD5zoTuP04eWn/B8qpg2StWrUKeCWhc+bMGTXXOpx/8cUXoV6Ojdv7X8SdM6Bds2bNmurYzp07q3lcXJzf1/v73/+u5pUqVVLzQB4Tp3/DNWvW2LK9e/eqY5966ik1P3TokN/rMJXbZyCcnwNatmyp5iNHjrRlbdu2Vcc6/ftpj2sgYwMd36tXL3Xsm2++qeaFhdv7XyS8z4CTp59+2pY5dTOuVq2amkdFRQV1Tb/3/vvv2zKnbuFOHco1X3/9tZrXr1/f7zkCRZdjAAAAAIAnUdACAAAAAIxEQQsAAAAAMBIFLQAAAADASBS0AAAAAAAj2dv0ekzFihVt2bhx49SxDz74oJrv379fzceMGWPLZs+e7f/iwlx0dLQtO3funAsrwZ9Zt26dmnupy3Hx4sXVvFWrVrbMjS7H4apo0aK2zKnrdig5dVcMRudRrSusU6fYEiVKqPkTTzyh5j/99FPeFwbPcvp6N3fuXDW/88471Vzb/4F2wg1kfDDmdvoc165dq+aHDx8O6JowW506ddR8+vTpaq69E0Kg+/Ts2bO27Msvv1THvvPOO2q+adMmNd+6dastGzx4sDo2kC7H6enpfo8tKNyhBQAAAAAYiYIWAAAAAGAkCloAAAAAgJEoaAEAAAAARvJMU6hLL71Uzd9//31bdvXVV6tjDxw4oOa33HKLmu/Zs8fP1YW3Ll26qLnWyOSaa64J9XLg4Morr1Tzzp07F/BKCp7WlEFE5OOPPy7gleD3nL5Wh4pTk6czZ86oeVpami2rXbt2QNfUGl856d27t5r/73//U/OxY8cGtBaEh3/+859qfscdd6h5MJqiLVu2TM2XL19uy5yaUAXSnMqJ01inuWfMmOH33DDH8OHD1bxfv35qXq1aNb/nPn36tJo//vjjav7111/bspSUFL+v92dKlSply4YMGRLQHOfPn7dlkyZNyuuSQoY7tAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEgUtAAAAAAAI3mmy/HTTz+t5lqXzC+//FIde91116l5ZmZm3hdmOK0DZ6NGjdSx06ZNU/OrrrpKzZOSkvK+MOSZU9fpl156Sc0rVqwYyuXk21dffaXm8fHxfs8RGxur5s2aNbNln3zyid/zIn8+++wzW7Zo0SJ1rNO/t9ahMTk5WR37/fffq/m6deuclui3mJgYNX/77bdt2c033xzQ3E6d+J999llbdu7cuYDmhtm07r2jRo1Sx2ZnZ6u5z+dTc23vOj2/BMKpC6vTOpwEOh7eVLVqVVsWjG7GIiKrVq2yZVOmTFHHBqtzcSAWL15syy6//PKA5tA6Gi9dujTPawoV7tACAAAAAIxEQQsAAAAAMBIFLQAAAADASBS0AAAAAAAjUdACAAAAAIzksyzL8mtgIe8W9/e//13NX375ZVtWoUIFdeyHH36o5k899ZSau9GxTFOqVCk1L126tC2799571bHdunVTc60zp9P1Xn/9dTV36kr6+eefq7nGz20aMoV9/zvRuvu999576tg6deqEeDV28+fPt2VaR70/49SZc/Xq1bYs0O5+3377rS2rXbt2QHMEg9v7X8TcM1DY/e1vf7NlTp20q1evHtDcY8aMsWVOz2eFndtnwNT9/+mnn9qya6+9Vh3r9BgvX75czXv16mXLzp496//iHGhrFgl83dq/mdNYp27+R44cUfOC5vb+FzH3DGjvVrB58+aA5vjPf/6j5rfddpsty8rKCmjuYLjxxhvVXOvCXKxYMXVsWlqammtd9LXXRqH2V2eAO7QAAAAAACNR0AIAAAAAjERBCwAAAAAwEgUtAAAAAMBIFLQAAAAAACNFur2AYFm6dKmaf/fdd7bslVdeUcc6dQlr3Lixms+ZM8eWPffcc+rYH3/8Uc2LFy9uy7p06aKO1ToKiohUq1ZNzbUOt/v371fHrl+/Xs2/+eYbW/Z///d/6tjC0g0Q/78VK1bYMje6GR89elTNJ0+ebMu++uqroFxz48aNtuyee+4JaI4rr7zSlvXs2VMdO2/evIDmBkREjh8/bsucOu4H2uW4ffv2tuyZZ55Rx7rRmRPuCLRbrdP4unXr+j1Hp06d1Fx7hwqnTvLBWPd1112njuX1C/6M09fkgv66+cILL6j5oEGD1LxIkSK2zKlDsdaxWUTk+++/93N17uIOLQAAAADASBS0AAAAAAAjUdACAAAAAIxEQQsAAAAAMJLPsizLr4EB/jJ+YRYZqffCGjFihJr369dPzStXruz3Nd98800179ixoy275JJL1LFa8xARkZkzZ6q51ijr448/dlhh4ebnNg2Zwr7/u3btquZaoyKn/R8M27dvV/OxY8eq+erVq/N9zSpVqqj5J598YsvKlSsX0Nxnz561ZU5NtQ4cOBDQ3IFwe/+LFP4z4CX33Xefmr/++uv5njs6OlrNz58/n++5Q8ntM2Dq/v/0009t2bXXXquOdXqMnT53bXwgY53GB2MdIiLLly+3ZU7NNbWv9YWJ2/tfxNwzUKxYMVvm1Ei2Xbt2an7q1Ck1b9u2rS3TXnv8mR49etgyp+Z9pUqVUnOtwayT0aNHq/nEiRP9nsMNf3UGuEMLAAAAADASBS0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADBS6NqdFmIlSpRQ87feekvNq1evruZ9+vTx+5r33nuv32Od1jF16lQ1N7VzMQJ39913q/m4cePUPJQdjefPn2/LHnroIXWsU4fAYKhdu7aaB9rRWJOVlWXLQtnNGAimFStW2DJtT8O79uzZY8saNWoU0ByBdLcNtBNuMOY+cuSImnfp0iWgtcCbMjMzbdmXX36pjnXqcuz07iPau6NMnz5dHTtkyBA1v+GGG2yZU53i5Pvvv1dzrfb4/PPPA5rbFNyhBQAAAAAYiYIWAAAAAGAkCloAAAAAgJEoaAEAAAAARqKgBQAAAAAYyfNdjps2bWrLXnrpJXVskyZN1Dw7O1vNjx07ZssWLVqkji1fvrya33nnnbasVatW6tjx48erOcLHFVdcoeY1atQI2TUfeOABNV+yZIktC2U3Y60ToIjI7NmzQ3bNuXPnhmxuwEl0dHRQ5vn5559tmdPzGbypZ8+etszpHRPq1q2r5t98843f13OaY86cOX7PYVmW32NFRJ5++umAxgNPPfWUmsfHx6t5+/bt1bxTp05+ZYE6f/68mj/zzDNq/sYbb6j5d999l++1mII7tAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEiebwr1/PPP2zKn5k+//PKLmj/77LNq7tRcKhCjR4+2ZWPHjlXHrl+/Xs3btm2r5p9//nme14Xwc/z4cTVPSUlR81A1gHJq/rR48WI1v/TSS/N9TafP5cUXX8z33ECgHnzwwaDMc+jQoaDMA2/ZuXNnQHkgtEaXIiI+ny+gXLN27Vo1D8ZrMXhXsWLFbFnNmjXVsXXq1An1cmy0pmbbt29Xx65YsSLUyzEWd2gBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEbyTJfjRx99VM2bN29uy7Kzs9WxTp0l33333bwv7C9MnDjRlt10003q2BYtWqh5jx491JwuxwjEe++9p+ZpaWn5nrtKlSpqXrt2bVs2e/ZsdWwwuhk7yczMVPPvv/8+ZNcEREQqVKhgy0qXLh3QHE4d+l977bU8rQnIq8cff1zNLcvyew6nsT179szTmhAe4uPj1XzUqFG27K677grKNS9cuGDLIiMDK60+/PBDW7Zu3bo8rylccYcWAAAAAGAkCloAAAAAgJEoaAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkz3Q57tSpk5pHRNhr9kWLFqljQ9nN2ElWVpYtO3/+fEBz3HvvvWr+3HPP2bLDhw8HNDfCR/HixdW8aNGial6sWDFbdv3116tj582bp+blypXzc3WhdezYMbeXgDDVq1cvW3bFFVcENMeOHTvU/MCBA3laE+AP7eu6z+cLaA5t/IwZM9SxR44cCWhuhJe+ffuqeSAdjTMyMtR88uTJan78+HFb9vzzz/t9PRGR1q1b2zK6HAeOO7QAAAAAACNR0AIAAAAAjERBCwAAAAAwEgUtAAAAAMBIFLQAAAAAACN5psvxd999p+Za11VTO5o6dQ/84osv1JyOxgiEU6fw+fPnq3mpUqVs2U033RTMJQXdrFmz1DzQroRAoLTzIiIycOBAv+fIzMxUc6cOnEAw1KlTR8215wzLstSxTrnWufi1117zf3EIOy+//LKa9+/f3+85nF7XDBgwQM1Pnz6t5o888ojf13Ryyy232LLHH3883/OGG+7QAgAAAACMREELAAAAADASBS0AAAAAwEgUtAAAAAAAI3mmKdS2bdvUvFevXrasfPnyoV6OTdOmTdW8R48etiwxMVEde+LECTV/6qmn8r4w4C907tzZ7SX8qX379qm51uhpw4YN6tg9e/YEc0mATe/evdW8cuXKfs+xcePGgHIgGG677TY1j42NtWVOzSudvPnmm7Zs586dAc2B8NKtWzc1j4jQ79Ht2rXLljk1kDpz5kye15VXX375ZYFf04u4QwsAAAAAMBIFLQAAAADASBS0AAAAAAAjUdACAAAAAIxEQQsAAAAAMJJnuhzPmTNHzW+++WZbduedd6pj//Of/6j5+vXr1bxYsWK27O6771bHVq9eXc21rmyHDh1Sxzp1dtu0aZOaw3uc9uivv/6q5mXKlLFlgXahLCycuhnfeuutav7tt9+GcjmA6tprr1XzYHSjf+edd/I9BxCoTp06qbllWX7P4TT26aefzsuSAL9lZGTYsmB1M65atWq+53jjjTfyvxBwhxYAAAAAYCYKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYCTPdDk+e/asmvfo0cOW9erVSx07fPhwNZ8wYULeF/b/OHUx++yzz2zZ7Nmz1bHHjh3L9zpgth07dqh5hQoV1HzgwIG2bOzYserY0qVL53ldF2VnZ6u5U2flCxcu2DKn/f/888+rOd2MUZjccsstal68eHG/5zh48KCaz5w5M09rAvyRlJSk5i1btlRz7eu99s4NIvprMRGRI0eO+Lk64DdTp05V89GjR6t57dq1bVnXrl3VsV999ZWaO31dHzRokJpr1qxZo+a7du3yew444w4tAAAAAMBIFLQAAAAAACNR0AIAAAAAjERBCwAAAAAwks+yLMuvgQ5NXYCC4Oc2DRkv7f+GDRuqefv27dX8kUceUfPDhw/bsqeeekodGxUVpeYbNmywZWlpaerYcOb2/hfx1hkIlu7du9uy119/XR0bHR3t97xt2rRR85SUFL/n8Bq3z4CX9n/58uXVfNWqVWp+7bXXqrn2b+L0OF133XVqvnPnTjVHbm7vf5HCfwZGjhyp5mPGjLFlRYsWDdk6MjIy1LxJkyZq7tSICrn91RngDi0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEh0OYYR3O7wx/6Hm9ze/yLhfQaKFCmi5m+99ZYtu+OOOwKae8uWLbasZcuW6tjCsA/c4vbn7qX937hxYzXftm2bmkdE6Pc+srOzbZlT1+LbbrtNzY8cOaLmyM3t/S9i7hnQutH/85//VMfGx8cHNPemTZts2XPPPaeOfe+99wKaG7nR5RgAAAAA4EkUtAAAAAAAI1HQAgAAAACMREELAAAAADASBS0AAAAAwEh0OYYR3O7wx/6Hm9ze/yLhfQZKly6t5sHo0Lp582Zb5tTlOJy5fQa8tP9jY2PV3KnL8VVXXaXmy5Yts2UDBgxQx9LNOH/c3v8i3joDMA9djgEAAAAAnkRBCwAAAAAwEgUtAAAAAMBIFLQAAAAAACNR0AIAAAAAjESXYxjB7Q5/7H+4ye39LxLeZ6BIkSJqPnr0aFs2atQodWxKSoqa33fffbZs//79AawuPLh9BsJ5/8N9bu9/Ec4A3EWXYwAAAACAJ1HQAgAAAACMREELAAAAADASBS0AAAAAwEg0hYIR3G6IwP6Hm9ze/yKcAbjL7TPA/oeb3N7/IpwBuIumUAAAAAAAT6KgBQAAAAAYiYIWAAAAAGAkCloAAAAAgJEoaAEAAAAARvK7yzEAAAAAAIUJd2gBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEYKq4J29uzZ4vP5JC0tLaD/r1WrVhIfHx/UtVStWlX69OkT1DmBP8P+R7jjDCCcsf8R7jgD3hVWBW04mDhxovh8vqAfPKAw2rBhg/h8PvXPxx9/7PbygJDbvXu33HXXXXLllVdKbGyslCtXTlq2bCkrV650e2lAgeM1EMJRRkaGjBgxQuLi4iQmJkaaNm0qH3zwgdvLKlCRbi8AwfPjjz/K008/LcWLF3d7KUCBGjx4sFx33XW5sho1ari0GqDg/PDDD3Lq1Cnp3bu3xMXFydmzZ2Xp0qXSsWNHSU5Oln79+rm9RKBA8BoI4apPnz6yZMkSGTJkiNSsWVNmz54t7dq1k5SUFLnhhhvcXl6BoKD1kGHDhkmzZs0kKytLjhw54vZygALTokUL6dKli9vLAApcu3btpF27drmygQMHSqNGjWTKlCkUtAgbvAZCOPrkk09k4cKFMnnyZBk2bJiIiPTq1Uvi4+Nl+PDhsmXLFpdXWDDC+keOV6xYIe3bt5e4uDiJioqS6tWry4QJEyQrK0sdv2PHDklISJCYmBipVq2aTJ8+3TYmIyNDxowZIzVq1JCoqCipXLmyDB8+XDIyMkL6uWzcuFGWLFkiL774YkivA+/w0v4XETl16pRcuHAh5NeBd3jtDFxUpEgRqVy5shw/frzArgnzeGn/8xoIeeGFM7BkyRIpUqRIrm9eRkdHS9++fWXr1q2yf//+kFy3sAnrO7SzZ8+WEiVKyNChQ6VEiRKyfv16GT16tJw8eVImT56ca+yxY8ekXbt20rVrV+nevbssXrxYBgwYIMWKFZP7779fRESys7OlY8eOsmnTJunXr5/UrVtXvvzyS5k6daqkpqbK8uXLHdeSnZ0tR48e9WvdpUqVkqJFi+b8PSsrSwYNGiQPPPCA1K9fP/AHAmHJK/tfROS+++6T06dPS5EiRaRFixYyefJkady4cWAPCMKOl87AmTNnJD09XU6cOCHvvPOOrF69Wrp16xbYA4Kw4pX9z2sg5JUXzsBnn30mtWrVkpIlS+Ya06RJExER2bVrl1SuXNnfh8RcVhiZNWuWJSLWvn37LMuyrLNnz9rGJCUlWbGxsda5c+dyssTEREtErBdeeCEny8jIsBo2bGhVqFDByszMtCzLsubNm2dFRERYH330Ua45p0+fbomItXnz5pysSpUqVu/evXP+vm/fPktE/PqTkpKSa/5p06ZZpUqVsg4dOpSz3nr16uXpMYJ3eXH/b9682fr73/9uzZw501qxYoX1zDPPWGXLlrWio6OtnTt35ufhggd58Qz8ft0XPx4REWF16dLFOnr0aF4eJniUV/c/r4HgLy+egXr16lmtW7e2fR67d++2RMSaPn16QI+RqcL6Dm1MTEzOf586dUoyMjKkRYsWkpycLHv27JEGDRrkfDwyMlKSkpJy/l6sWDFJSkqSAQMGyI4dO6RZs2by1ltvSd26daVOnTq5fn+jdevWIiKSkpIiCQkJ6loqVqzod0ey36/r119/ldGjR8uTTz4p5cuX9+8TB8Qb+z8hISHXnB07dpQuXbrI1VdfLSNHjpT333/frzkRnrxwBi4aMmSIdOnSRQ4ePCiLFy+WrKwsyczM9Gs+hCcv7H9eAyE/vHAG0tPTJSoqyjYmOjo65+PhIKwL2t27d8uoUaNk/fr1cvLkyVwfO3HiRK6/x8XF2Trn1apVS0RE0tLSpFmzZrJ371755ptvHL+oHjp0yHEt0dHR0qZNm4A/h1GjRkmZMmVk0KBBAf+/CG9e2P+aGjVqyB133CHLli2TrKwsKVKkSFDmhfd46QzUqVNH6tSpIyK/NQRp27atdOjQQbZt2yY+ny/P88K7vLD/eQ2E/PDCGYiJiVF/P/fcuXM5Hw8HYVvQHj9+XBITE6VkyZIyfvx4qV69ukRHR8vOnTtlxIgRkp2dHfCc2dnZUr9+fZkyZYr68T/7GfasrCw5fPiwX9cpU6aMFCtWTPbu3SszZsyQF198UQ4ePJjz8XPnzsn58+clLS1NSpYsKWXKlAnsE4HneWH//5nKlStLZmamnDlzxvZ7JYCI989Aly5dJCkpSVJTU6V27dp+zYvw4YX9z2sg5IcXzoCISKVKleTAgQO2MT/99JOI/FaIh4OwLWg3bNggv/76qyxbtkxatmyZk+/bt08df/DgQTlz5kyu786kpqaKiEjVqlVFRKR69ery+eefy0033RTwd8T3798v1apV82tsSkqKtGrVSg4cOCDZ2dkyePBgGTx4sG1ctWrV5JFHHqHrH2y8sP//zPfffy/R0dFSokSJgNaB8OH1M3Dxx8z+eJcBEPHG/uc1EPLDC2dARKRhw4aSkpIiJ0+ezPUN/G3btuV8PByEbUF78ccQLcvKyTIzM+WVV15Rx1+4cEGSk5Nl6NChOWOTk5OlfPny0qhRIxER6dq1q6xatUpee+0123v/paenS3Z2tuMbfuflZ+fj4+Pl7bfftn181KhRcurUKXnppZekevXqfs2J8OKF/S8icvjwYduP9nz++efyzjvvyG233SYREWH9zmT4E145A4cOHZIKFSrk+vj58+dl7ty5EhMTI1dddZVfcyK8eGH/8xoI+eGFMyDy20/jPP/88zJjxoyc96HNyMiQWbNmSdOmTcOjw7GEcUGbkJAgpUuXlt69e8vgwYPF5/PJvHnzcm3s34uLi5NJkyZJWlqa1KpVSxYtWiS7du2SGTNm5LTO7tmzpyxevFj69+8vKSkpcv3110tWVpbs2bNHFi9eLGvWrHF8K5G8/Ox8uXLlpFOnTrb84ncjtY8BIt7Y/yIi3bp1k5iYGElISJAKFSrI119/LTNmzJDY2Fh59tlnA54P4cMrZyApKUlOnjwpLVu2lMsuu0x+/vlnefPNN2XPnj3ywgsv8FMKUHlh//MaCPnhhTMgItK0aVO56667ZOTIkXLo0CGpUaOGzJkzR9LS0mTmzJkBz2cst9oru+GP7bo3b95sNWvWzIqJibHi4uKs4cOHW2vWrLG1xL7YAn779u1W8+bNrejoaKtKlSrWtGnTbNfIzMy0Jk2aZNWrV8+KioqySpcubTVq1MgaN26cdeLEiZxxf2zXHUy0rIfGi/v/pZdespo0aWKVKVPGioyMtCpVqmT16NHD2rt3b77nhvd48QwsWLDAatOmjXXppZdakZGRVunSpa02bdpYK1asyPfc8BYv7n8Nr4HgxKtnID093Ro2bJhVsWJFKyoqyrruuuus999/Pyhzm8JnWQ7figAAAAAAoBDjF8wAAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgpEh/B/p8vlCuA/hTbr9dMvsfbnJ7/4twBuAut88A+x9ucnv/i3AG4K6/OgPcoQUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGCnS7QUAAFCYtW3bVs179uxpy+6991517K5du9Q8LS3NlnXu3NnvtQEAEO64QwsAAAAAMBIFLQAAAADASBS0AAAAAAAjUdACAAAAAIxEQQsAAAAAMJLPsizLr4E+X6jX4mljx45V8w0bNviVhTs/t2nIsP/hJrf3v0h4n4GtW7eqeZMmTfI999mzZ21Znz591LFLly7N9/VM5fYZCOf9X9BefvllNX/zzTfV/OOPPw7lcgoFt/e/CGegMBs8eLCa16lTx5YlJSUFNHdEhP3eZ40aNdSx3333XUBzB+KvzgB3aAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkCloAAAAAgJFoCpUPrVq1smVjxozxe6yTwv5YO30uTrlTk6tAml+53RChsP+bwNvc3v8i4X0Gfv75ZzUvX758vufWHtf//Oc/6tibb74539czldtnwEv7v169emr+z3/+U82dmpEtX74832t59NFHbdlzzz2njnVq/nT99dfnex2Fndv7X8RbZ6Aw0Z5HXnvtNXVs3bp11dypSVMw9o327/7444+rYydNmpTv6zmhKRQAAAAAwJMoaAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkCloAAAAAgJEi3V6AybSuvoF0MzaB9vmkpKQENIdT52c65iEYLrnkEjUvXry4LUtPT1fHxsTE5Hsdx44dU/OMjIx8zw13vfDCC2r+7LPPhuR6xYoVU/PISP0p+8KFCyFZB7wpPj5ezXv06KHmnTp1UnOt4+qPP/4Y0Fq0DsUREfq9lmbNmql5oO+wALihb9++at6vXz9b1qhRo1AvJ1+WLFni9hJsuEMLAAAAADASBS0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADASXY5/x6lTnlPu1L03EOPGjcv3HIEIh88R7hkxYoSa33jjjfme26krds2aNdW8SpUqtuzgwYPq2Msuu0zNLcvyc3UiO3fuVPPrrrvO7zlQOE2dOlXNb731VlsWjE73N9xwg5rXqVNHzb/66qt8XxNwUqJECTV/8cUXbVmXLl1Ctg6n7sfDhw9Xc7ocI9TKly9vy5544gl17KBBg9Q8kNcZcMYdWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYKSwbAoVaAOkYDT5cGqMNHbs2HzPHYhQfo5ODRgK+nOEe5555hm3l/CnnJo/OTWcCkTFihXzPQcKp/j4eDWvV69eSK63fft2Nf/+++9Dcj2El6uvvjoo8wTyddOpoVN0dHS+13Hu3Ll8zwH8mV69eqn5Y489Zsvq1q2b7+u98847aq41YhMR2bhxo99z9+3bV82Tk5P9nqMw4g4tAAAAAMBIFLQAAAAAACNR0AIAAAAAjERBCwAAAAAwEgUtAAAAAMBInu9yrHXYTUxMVMea2s04kK7NoexmfOONN+Z7bhQ+l1xyiZp/+OGHfs9hWZaap6am2rL09HR17ObNm9U8LS1NzatWrer32EB8/fXXar5ly5Z8zw13OXUtXrNmjZqXK1cuJOto0KCBmo8fP17NR48ereZnz54N2prgHY0aNSrwazp1Vr7tttvyPffChQvzPQfCi1OH7okTJ6r5P/7xDzUvWrSo39f84Ycf1Lx79+627Msvv1THBvo1Xat3pk6dGtAcc+fOtWX/+9//ApqjIHCHFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJM93OdY6/YZSKLsZO3H6HIPR0VhDN2NvqlChgppPnjxZzZ06sWoeeOABNde6Uzp1OQZCzakTa6i6GTtx6pzp1GmzcePGat6pUydbdvz48bwuC8jl448/tmVNmjRRx86ZMyfUywH81r9/fzUfPnx4yK45a9YsNd+2bVvIrqk9Z8TGxqpjDx8+rOaTJk2yZefPn8/fwkKAO7QAAAAAACNR0AIAAAAAjERBCwAAAAAwEgUtAAAAAMBIFLQAAAAAACMZ1+XYqXNvQXczFgldt1+nzzElJSUk1xMR2bBhg5qPGzcuZNdE4fLkk0+qeY8ePfI9t1NXvejoaFtGl2MgMC1atFDz5ORkW9atW7dQLweF3FdffaXmt9xyS0DzPPfcc8FYTr5lZma6vQQYpnbt2iGbe8mSJWo+YcKEkF0zMTFRzbXnBqduxk7nf8+ePXlfWAHiDi0AAAAAwEgUtAAAAAAAI1HQAgAAAACMREELAAAAADCSz7Isy6+BPl+o1+KXsWPHqnkom0I5NUwKRlMorQFUKJs/OXFq/qQ93k7/Bk55MPi5TUOmsOz/YNAaxYiIPPjggwW8Et26devUfPz48Wq+adOmUC6nUHB7/4t46ww46d69u5q/8cYbIbum9rgG6987IyPDliUkJKhjd+3aFZRrhorbZ8BL+9+pMdjChQsLeCWBcWoY6NR00Evc3v8i3joDTp/LhQsXAprn22+/tWWhbDjlxGl/ZGdn27KZM2eqY/v16xfUNQXbX50B7tACAAAAAIxEQQsAAAAAMBIFLQAAAADASBS0AAAAAAAjUdACAAAAAIwU6fYCApWYmBiyuQPp9BsorZuxSGi7MwfCaR1a7tT12elzdBqP0GvTpo0tc+pmHIwuik6dA7/66is1r1atmi276aab1LGXX365mjdt2lTNT506peaAk507d6r50KFD/Z5j2bJlal62bFk1175u3n777erYFi1aqHnRokXVPDo62pY5nZfC3uUYwfPZZ5+p+TfffKPmdevWDeVy/LZ37163lwCPGDVqlJo7vQ46fPiwmg8ePDhoa/q94sWLq/mLL76o5lo3YxH99feQIUPyuKrCjTu0AAAAAAAjUdACAAAAAIxEQQsAAAAAMBIFLQAAAADASBS0AAAAAAAj+Sw/W5s6dS8taE4dh0PZLdip+7HWcdmp06+X3HjjjWoeym7GwejAmx+FZf8Hw4ABA9T8gQceUPN3331XzZcuXer3Nb/++ms117q2rlu3Th2bmZmp5ldccYWaO3UlNJHb+1/EW2fAVO+9956a33rrrX7PkZCQoObbtm3L05oKittnIBz2v9PX0o4dO6p5jRo1bFmPHj3UsT/99JOax8fH+7k6kfT0dDWPjY31ew5Tub3/Rcw9A8WKFbNlM2bMUMc67d+5c+eq+f3335/3hf2J559/Xs2dOhQ7/dvccccdtszpNV1h91dngDu0AAAAAAAjUdACAAAAAIxEQQsAAAAAMBIFLQAAAADASMY1hUpJSVHzcGjGFEpOja+0Rk+hbP7kxO2GCIVl/3vNsGHDbNmkSZPUsbt371bzpk2bqrlTAxETub3/RTgDhYHT81/Lli39noOmUHnD/s8fp2Y2U6dO9XsOmkK5y9QzoDUv27NnT0Bz1K5dW82/++67PK3p96666ipbtnLlSnVslSpV1Pyjjz5S806dOtmyEydO+L+4QoSmUAAAAAAAT6KgBQAAAAAYiYIWAAAAAGAkCloAAAAAgJEoaAEAAAAARop0ewGBuvHGG9U8nLsfO3Uo1owdOzZ0C0FQxcfH27IFCxaoY1evXq3mw4cPD+qagq1Hjx62zNROijBHqVKl1Dw6OlrNDx8+rObZ2dn5XkuZMmVs2YsvvqiObdGiRUBzX7hwwZadP38+oDkAwGvceJ3RoEEDNV+7dq0tK1eunDp248aNau5UG4UT7tACAAAAAIxEQQsAAAAAMBIFLQAAAADASBS0AAAAAAAjUdACAAAAAIxkXJdjJ04dvrSuvomJiepYNzoib9iwwZZ9+OGH6lg6FIeXFStW2LKqVauqY4cNG5bv68XExKh5+fLl1bx06dK27O9//7s69oEHHlDzsmXL2jLLstSxnTp1UvP09HQ1B5y8+uqrat6tWzc1HzlypJo/99xztkw7FyIitWvXVvPHH3/clrVv314dG6gdO3bYsp07dwZlbiAQK1euVPOpU6f6PYdTF/Kbb75ZzT/44AO/54Z3Pfnkk7bM6XXG3Llz1fx///tfvteRlJSk5trroFWrVqljtXeGwG+4QwsAAAAAMBIFLQAAAADASBS0AAAAAAAjUdACAAAAAIxEQQsAAAAAMJJnuhw70ToDO3ULdqPL8bhx42yZ1vkY4UfrUOrU5XjOnDlq/tNPP9my9evXq2OdOquWK1dOzbVurj6fTx3r1FEwOzvblm3ZskUde+zYMTUHQm3ixIlqfuedd9oypy7HNWvWVHPtzDidFydffPGFmk+ePDmgeYBQiYqKyvccTs8vTmcO4aVx48Zqfsstt/g9x4kTJ9T8/Pnzal6sWDFbdsUVV6hjnbocnz592pZNmjQpoPWBO7QAAAAAAENR0AIAAAAAjERBCwAAAAAwEgUtAAAAAMBInm8KpTV6GjNmTMiu59TQSWv+9GfjgaNHj/o9tnz58n7nDRo0UMcG2ogmEE4Nnfr27WvLVqxYEbJ1AHkREaF/77dJkyYFuo4LFy6o+WOPPabm69atC+VyAL8F8nwG5IXT6yCnxpbBoDWA2rNnT0BzDBkyxJZt2rQpr0sKW9yhBQAAAAAYiYIWAAAAAGAkCloAAAAAgJEoaAEAAAAARqKgBQAAAAAYyfNdjlNSUgr0eh9++KGa080Ygerfv78tGzlypDr29ttvV/Mrr7zSliUkJKhjg9Hl+M0331TzJUuWqHl6enq+rwmYzOfz2bLTp0+rY3v27KnmdDNGYXfkyBE1114zJSYmBjS3UxdyQET/Ghvo2GeeeUbNhw8f7vfc3bp1U3On10cIDF8FAAAAAABGoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkShoAQAAAABG8kyX41atWhX4NbXOxWPHji3wdSB8HDt2TM3nzZtXwCsBzLVs2TI1d+pCGQz79u1T8y1bttiyKVOmqGN37doVzCUBritSpEi+5+jdu7eaL1y4MN9zw3yBvIODUyf52NhYNdc60m/cuFEdSzfj0OIOLQAAAADASBS0AAAAAAAjUdACAAAAAIxEQQsAAAAAMJJnmkJpDZpCbdy4cQV+TQBA/rz77rtqPmfOHDV3ajqjcWo49dhjj6l5Wlqa33MDpoqJiVHza6+9toBXAq86e/asmp85c8aWFS9eXB1bqlSpgK65fft2W9ahQ4eA5kBwcIcWAAAAAGAkCloAAAAAgJEoaAEAAAAARqKgBQAAAAAYiYIWAAAAAGAkn2VZll8Dfb5QryUkWrVq5Vf2Z8aOHRuUtSDv/NymIWPq/oc3uL3/RTgDcJfbZ4D9HxpaV/A777wzoDnef/99Nb/tttvytKbCyO39L2LuGbjvvvts2WuvvRbQHE899ZSaz5o1y5b98MMPAc0N//zVGeAOLQAAAADASBS0AAAAAAAjUdACAAAAAIxEQQsAAAAAMBIFLQAAAADASJ7vcgxvcLvDH/sfbnJ7/4twBuAut88A+x9ucnv/i3AG4C66HAMAAAAAPImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEbyWZZlub0IAAAAAAACxR1aAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRwqqgnT17tvh8PklLSwvo/2vVqpXEx8cHdS1Vq1aVPn36BHVO4M+w/xHuOAMIZ+x/hDvOgHeFVUHrZTt37pSOHTtKmTJlJDY2VuLj4+Vf//qX28sCCgT7H+Fq9+7dctddd8mVV14psbGxUq5cOWnZsqWsXLnS7aUBBSIjI0NGjBghcXFxEhMTI02bNpUPPvjA7WUBBeLTTz+VgQMHSr169aR48eJyxRVXSNeuXSU1NdXtpRWoSLcXgPxbu3atdOjQQa655hp58sknpUSJEvLdd9/Jjz/+6PbSgJBj/yOc/fDDD3Lq1Cnp3bu3xMXFydmzZ2Xp0qXSsWNHSU5Oln79+rm9RCCk+vTpI0uWLJEhQ4ZIzZo1Zfbs2dKuXTtJSUmRG264we3lASE1adIk2bx5s9x1111y9dVXy88//yzTpk2Ta6+9Vj7++OOg31kurChoDXfy5Enp1auXtG/fXpYsWSIREdx0R/hg/yPctWvXTtq1a5crGzhwoDRq1EimTJlCQQtP++STT2ThwoUyefJkGTZsmIiI9OrVS+Lj42X48OGyZcsWl1cIhNbQoUNl/vz5UqxYsZysW7duUr9+fXn22WfljTfecHF1BSesX/2tWLFC2rdvL3FxcRIVFSXVq1eXCRMmSFZWljp+x44dkpCQIDExMVKtWjWZPn26bUxGRoaMGTNGatSoIVFRUVK5cmUZPny4ZGRkhORzmD9/vvzyyy8yceJEiYiIkDNnzkh2dnZIrgVvYf8j3HnhDGiKFCkilStXluPHjxfYNWEeL+z/JUuWSJEiRXJ94yY6Olr69u0rW7dulf3794fkuvAGL5yBhISEXMWsiEjNmjWlXr168s0334TkmoVRWN+hnT17tpQoUUKGDh0qJUqUkPXr18vo0aPl5MmTMnny5Fxjjx07Ju3atZOuXbtK9+7dZfHixTJgwAApVqyY3H///SIikp2dLR07dpRNmzZJv379pG7duvLll1/K1KlTJTU1VZYvX+64luzsbDl69Khf6y5VqpQULVpURETWrVsnJUuWlAMHDkinTp0kNTVVihcvLj179pSpU6dKdHR03h4ceB77H+HOC2fgojNnzkh6erqcOHFC3nnnHVm9erV069YtsAcEYcUL+/+zzz6TWrVqScmSJXONadKkiYiI7Nq1SypXruzvQ4Iw44UzoLEsS3755RepV6+eX/N5ghVGZs2aZYmItW/fPsuyLOvs2bO2MUlJSVZsbKx17ty5nCwxMdESEeuFF17IyTIyMqyGDRtaFSpUsDIzMy3Lsqx58+ZZERER1kcffZRrzunTp1siYm3evDknq1KlitW7d++cv+/bt88SEb/+pKSk5Px/V199tRUbG2vFxsZagwYNspYuXWoNGjTIEhHr7rvvzs/DBY9h/yPcefEM/H7dFz8eERFhdenSxTp69GheHiZ4lBf3f7169azWrVvbPo/du3dbImJNnz49oMcI3ubFM6CZN2+eJSLWzJkz/X1ojBfWd2hjYmJy/vvUqVOSkZEhLVq0kOTkZNmzZ480aNAg5+ORkZGSlJSU8/dixYpJUlKSDBgwQHbs2CHNmjWTt956S+rWrSt16tSRI0eO5Ixt3bq1iIikpKRIQkKCupaKFSv63ZXv9+s6ffq0nD17Vvr375/T1bVz586SmZkpycnJMn78eKlZs6Zf8yK8sP8R7rxwBi4aMmSIdOnSRQ4ePCiLFy+WrKwsyczM9Gs+hCcv7P/09HSJioqyjbn40znp6el+zYnw5IUz8Ed79uyRhx9+WJo3by69e/f2az4vCOuCdvfu3TJq1ChZv369nDx5MtfHTpw4kevvcXFxUrx48VxZrVq1REQkLS1NmjVrJnv37pVvvvlGypcvr17v0KFDjmuJjo6WNm3aBPw5XDyM3bt3z5Xfc889kpycLFu3buUFPVTsf4Q7L5yBi+rUqSN16tQRkd+a4rRt21Y6dOgg27ZtE5/Pl+d54V1e2P8xMTHq7yaeO3cu5+OAEy+cgd/7+eefpX379lKqVKmc3y8PF2Fb0B4/flwSExOlZMmSMn78eKlevbpER0fLzp07ZcSIEXlqLJOdnS3169eXKVOmqB//s9/jyMrKksOHD/t1nTJlyuT8AnhcXJzs3r1bLr300lxjKlSoICK//cw/8Efsf4Q7r5wBJ126dJGkpCRJTU2V2rVr+zUvwodX9n+lSpXkwIEDtjE//fSTiPz2HAFovHIGLjpx4oTcdtttcvz4cfnoo4/Cbu+HbUG7YcMG+fXXX2XZsmXSsmXLnHzfvn3q+IMHD8qZM2dyfXfm4psWV61aVUREqlevLp9//rncdNNNAX9HfP/+/VKtWjW/xqakpEirVq1ERKRRo0bywQcfyIEDB3K9aDl48KCIiON3iRDe2P8Id145A04u/qjlH+8yACLe2f8NGzaUlJQUOXnyZK7GUNu2bcv5OKDxyhkQ+e0nEjp06CCpqamybt06ueqqqwK6theEbUF78Ta8ZVk5WWZmprzyyivq+AsXLkhycrIMHTo0Z2xycrKUL19eGjVqJCIiXbt2lVWrVslrr71me++/9PR0yc7Otv24wkV5/dn5rl27yrPPPiszZ87M+Rl9EZHXX39dIiMj//JFD8IT+x/hzitn4NChQzk/kXDR+fPnZe7cuRITExOWL2zw17yy/7t06SLPP/+8zJgxI+d9aDMyMmTWrFnStGlTOhzDkVfOQFZWlnTr1k22bt0qK1askObNm/s1h9eEbUGbkJAgpUuXlt69e8vgwYPF5/PJvHnzcm3s34uLi5NJkyZJWlqa1KpVSxYtWiS7du2SGTNm5LTO7tmzpyxevFj69+8vKSkpcv3110tWVpbs2bNHFi9eLGvWrJHGjRur8+f1Z+evueYauf/+++X//u//5MKFC5KYmCgbNmyQt956S0aOHBl2P3IA/7D/Ee68cgaSkpLk5MmT0rJlS7nsssvk559/ljfffFP27NkjL7zwgpQoUSLgOeF9Xtn/TZs2lbvuuktGjhwphw4dkho1asicOXMkLS1NZs6cGfB8CB9eOQOPPvqovPPOO9KhQwc5evSovPHGG7k+3qNHj4DnNJJb7ZXd8Md23Zs3b7aaNWtmxcTEWHFxcdbw4cOtNWvW2FpiJyYmWvXq1bO2b99uNW/e3IqOjraqVKliTZs2zXaNzMxMa9KkSVa9evWsqKgoq3Tp0lajRo2scePGWSdOnMgZ98d23fmRmZlpjR071qpSpYpVtGhRq0aNGtbUqVODMje8g/2PcOfFM7BgwQKrTZs21qWXXmpFRkZapUuXttq0aWOtWLEi33PDW7y4/y3LstLT061hw4ZZFStWtKKioqzrrrvOev/994MyN7zFi2fg4lsKOf0JFz7LcvhWBAAAAAAAhViE2wsAAAAAACAvKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGCnS34E+ny+U6wD+lNtvl8z+h5vc3v8inAG4y+0zwP6Hm9ze/yKcAbjrr84Ad2gBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaioAUAAAAAGImCFgAAAABgJApaAAAAAICRKGgBAAAAAEaKdHsBsGvbtq2aDxw4UM1vvvlmNb/++utt2c6dO/O+MCCPqlatquYLFiywZU8//bQ6duXKlcFcEgAAADyAO7QAAAAAACNR0AIAAAAAjERBCwAAAAAwEgUtAAAAAMBIFLQAAAAAACPR5dhlbdq0sWVvv/22Ova7775T8/r166v5t99+m/eFAXkQHR2t5vPmzVPzb775xpa99957QV0TAADwtksuuUTNH3vsMVt2++23q2OvueYaNT906JCaJycn27KDBw+qY2fOnKnm58+fV3MEhju0AAAAAAAjUdACAAAAAIxEQQsAAAAAMBIFLQAAAADASBS0AAAAAAAj+SzLsvwa6POFei2edvnll6v5V199Zcs++ugjdez999+v5ocPH877wgzh5zYNGfa/fx588EE1Hzp0qJpfd911tuz06dNBXZMXuL3/RQr/GahRo4aaR0VF2bLq1aurYzt27Kjm9913n9/rOH78uJo/9dRTav7mm2/aMqeOmuHM7TNQ2Pe/E+21x6JFi9SxzZs3D2juTz/91Ja9++676lhtn4uI7N+/35bR9dXO7f0vUnjOgPY1XcT5tXOjRo1CuRy//fjjj2q+YMECW/b666+rY8P53Uv+6gxwhxYAAAAAYCQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYCS6HAeZUze1adOmqfmuXbts2YABA4K5JE9wu8Mf+99O6yq7fft2dewzzzyj5pMmTQrqmrzK7f0v4s4ZmDBhgi27/vrr1bGNGzdW8+LFi9syp8czIyNDzVeuXKnmt9xyiy0rWbKkOtbpmp9//rktKyxdOQsTt8+Aqc8BX3zxhS2rVauWOjYtLU3NL730UjV32uuBWL9+vS3r27evOvZ///tfvq9nKrf3v0jhOQPFihVT86VLl6p5vXr1bNlLL70U0DXLlSun5v3797dll1xyiTq2aNGifl/P6Sxqzzki4dH9mC7HAAAAAABPoqAFAAAAABiJghYAAAAAYCQKWgAAAACAkWgKFWTz5s1T86uuukrNaf7hH7cbIrD/7ebPn2/LqlSpoo5t2bKlmmdlZQV1TV7l9v4XCe0Z0BqMiYhs2rTJljk153Cyf/9+WzZr1ix17NmzZ9X83XffVfMNGzbYsvLly6tjnf4NU1NTbZnT80U4c/sMmPoc0KlTJ1vm1LRm1apVal6qVCk1T05OtmWtW7f2f3EOfvnlFzW/55571Fw7h17j9v4XMfcMFLT27dur+T//+U811xoaOjW+cjqjnTt3VvPz58+ruYloCgUAAAAA8CQKWgAAAACAkShoAQAAAABGoqAFAAAAABiJghYAAAAAYCS6HPuhdOnSav6vf/3Llt1xxx3q2DFjxqj51KlT876wMOJ2h79w3v833HCDmv/nP/+xZU7dWb/77rugrincuL3/Rdw5Aw8++KAte/XVV9Wxw4YNU/OVK1faMqf96NTN9YsvvlDzyy67zJY5PU5btmxR8zZt2tiy/6+9u4/Vev7/AP5JuiEhlptlbZZly11ITHMzdy2knXVDNZRKK8PcJYbclZtkjKLEGRm5W82Y3ERDrIaZ1DCZu7AoxdRSp/P74/fbfrbP6/P9Xtc553Jd73Mejz+fe+39eXfO+33O9fKZ19m2bVtY25ZV+w60hd8BPXv2DPNZs2aFeTRZ9c8//wxrV69eHebR1PLevXuHtVu3bg3zK664Isznz58f5imq9vnPsrZxB6ohmtJ90kknlbXGkUceGeZF9y5FphwDAADQKmloAQAASJKGFgAAgCRpaAEAAEiShhYAAIAk7VrtDdSS9u3bh/njjz8e5oMGDcplY8aMCWufe+65Ju8Lqmny5Mlh/uyzz+Yy04xpSfX19bls+fLlYW3R2StnYnCnTp3CPJpmXK7XX389zE005t/WoUOHML/uuuvCPJpmnGVZ9tNPP+Wyyy67LKx9+eWXS9xdlk2cODHMZ8yYEebXX399mK9atSqXrVixouR9wL9h48aN1d5Cq+ANLQAAAEnS0AIAAJAkDS0AAABJ0tACAACQJEOh/mHevHlhPmTIkDC/4YYbcpnhT6Sqrq4uzEeMGBHmZ5xxRiW3A9mOHTty2Zo1ayr2vPXr14f5hAkTwnzOnDm5rGiw1IABA8J8v/32K3kf0BImTZoU5kUDAItEgzE///zzJu3pn+bOnRvmvXr1CvNrrrkmzJcsWZLL+vbtG9Z+9913pW0OWtgvv/xS7S20Ct7QAgAAkCQNLQAAAEnS0AIAAJAkDS0AAABJ0tACAACQpDY55XjBggVhPnr06DC/9957y8ohRZ07dw7zoqmy7733XiW3AzWjvr4+zA877LBcdtVVV4W1Z555ZpgvXbo0l11++eVh7bJlywp2CLFLL700l912221h7c6dO8P8xhtvDPNKThyP3HTTTWG+zz77hPnYsWNzWTT5OMuKp/avW7euxN3Bfxb9vsiyLBs2bFjJa2zatCnMt27d2pQttSre0AIAAJAkDS0AAABJ0tACAACQJA0tAAAASdLQAgAAkKR2jY2NjSUVtmtX6b1URO/evXPZxx9/HNYuXrw4zMeNGxfmf//9d5P3RXlKPKYVk+r5L8cjjzwS5kVTHu+8885Kbod/qPb5z7K2cQdawvjx48O8aCr+nnvumcu2b98e1hZNP54/f36Ju0tXte9Aquf/s88+y2VF01ZXrFgR5ieeeGKL7unfEn2mGzx4cFh7yy23hPn06dNbcktNVu3zn2Xp3oF/W9euXcP80UcfDfMLLrig5LUffvjhML/yyitLXiNV/+0OeEMLAABAkjS0AAAAJElDCwAAQJI0tAAAACRJQwsAAECSWv2U47Vr1+ayHj16hLXHHHNMmK9Zs6ZF99RU0cTmLMuy8847r+Q13nnnnTAvmvxcK6o94S/V81+OTz/9NMxffPHFMG+JKcf9+/fPZTfffHNYe+qpp4Z5Q0NDmA8cODCXFU3xrHXVPv9Z1jbuQCUddNBBYf7EE0/kstNOO62stSdMmBDm9fX1Za1Ty6p9B2r9/Hfv3j3MV65cmct69uwZ1k6dOjXMZ86c2fSNVdGxxx6by6KvR5Zl2fr168P8nHPOyWWffPJJ8zbWBNU+/1lW+3egVjz00ENhPnny5JLXeOGFF8K86Gf9n3/+WfLaqTLlGAAAgFZJQwsAAECSNLQAAAAkSUMLAABAknat9gYq7eCDD85ll112WVhbK8Ofhg8fHuZPPfVUmHfs2LHktVevXh3mxx9/fJhv3bq15LVJ23fffVextfv16xfm0eCDLl26hLVFwzxOPPHEML/44otzWapDoUjfjz/+GOZnn312LpsxY0ZYe/XVV4f5nDlzSt5HaxoUxf878sgjwzwaALVly5awdunSpS26p2qLBh3ecccdYW3RMMK6urpcVo2hUFTXHnvsEeazZs3KZUOHDi1r7Y0bN+ayW2+9NaxtC8OfmsobWgAAAJKkoQUAACBJGloAAACSpKEFAAAgSRpaAAAAkpTclONOnTqF+YIFC8I8mh62cOHCFt1TKYr2PXbs2FxWzsTKLMuyDz/8MMyjaWgDBw4Ma/fcc88wN+W47dhvv/3CvHfv3iWvUXTOZ86cGeZffvllLhs1alRY+9tvv4X59OnTw7xbt25hDrVkx44duWzKlClh7QknnBDmJ510UpjPnz8/l7Vv377kWtLXrl27XPbXX3+Fta1tem9DQ0Mumzt3blg7ZsyYMD/11FNbcEfUuqK/sjBv3rwwP//880teO+pHsizLRo8encu++OKLktflf3lDCwAAQJI0tAAAACRJQwsAAECSNLQAAAAkSUMLAABAkpKbclw0uXTo0KFhfs899+Sy33//vUX3VIphw4aF+ezZs3PZ999/H9ZG/5Ysy7LHHnsszLt27ZrLNmzYULRF2rjFixeH+dSpU8O8Q4cOuaxv375hbVF+3HHH5bKiacZF3n333TAfMmRIWetArRs5cmSYF/3OaGxszGV33HFHWGvKcesUnYEoayt+/vnnMC/6HDVt2rRcduaZZ4a1b775ZtM3xr9q9913D/Oin4MjRowoee1yphlnWZa98cYbJa9NMW9oAQAASJKGFgAAgCRpaAEAAEiShhYAAIAkJTcUqlwzZ86s2NqdOnXKZXPmzAlrhw8fHubRgI4HH3wwrC0aZnXWWWeF+bx583LZ22+/HdYaFsXXX38d5nvttVeYn3vuubmsc+fOYe3KlSvLemY56urqwnznzp3NXhtqybp165q9xm677RbmvXr1CvO1a9c2+5nUln333TfMBw0aFOavvfZaJbdTE7755pswb9++fS674YYbwlpDoWpT9DPviSeeCGuLPqsXiT6Xjxo1Kqx1PirLG1oAAACSpKEFAAAgSRpaAAAAkqShBQAAIEkaWgAAAJKU3JTjokm/n3zySZgfe+yxueytt95qkb2ccsopuWzMmDFh7ebNm8N87ty5uazo39itW7cwj6YZZ1mWbd++PZdNmzYtrN2xY0eY03asWrUqzDdu3Bjm1113XS57/PHHW3RP/9SxY8cwv+iii8I82h+krEePHs1eo0OHDmHevXv3MDflOA3Lli0L8zVr1uSyPn36hLUHHHBAS26p1SqaEk1tOvnkk3NZudOMN23aFOYjR47MZaYZV4c3tAAAACRJQwsAAECSNLQAAAAkSUMLAABAkjS0AAAAJCm5Kcfbtm0L86JJrJWcchxNUf3hhx/C2qOPPjrMo4nGF198cVh77bXXhvn+++8f5hdeeGEuW758eVgLX331VZi/9tprYT569Ohc1rlz57D2119/bfrG/k80TfA/rT1//vxmPxOqoWia8ZIlS5q99pYtW8I8moZLOhoaGsK8sbGx5DUmTJgQ5vX19U3aU0pMeE5f0V8Cef7555u99tSpU8PcROPa4Q0tAAAASdLQAgAAkCQNLQAAAEnS0AIAAJAkDS0AAABJSm7KcZHHHnsszJ9++ulctn79+rB24cKFYd6nT58wP/3003PZOeecE9Z27do1zJcuXZrL+vbtG9a+//77YX7EEUeE+ddffx3mUI7Zs2eH+eDBg3NZ0dnduXNnmN922225rFOnTmHtpEmTwvyWW24J86KJ6LQtBx54YC7bddf4V1/RlPpKiu5M0VTOXr16hfkuu8T/bTq6d7fffntY+8cffxTskJTddddduezJJ58Maw8//PAwr6urC/NFixY1fWM1ZsSIESXXtsTUXFpe0c/BPfbYo9lr9+/fP8w3b96cy2rpfBx66KG57LTTTgtrr7rqqjB/5plnctmtt97arH1Vgje0AAAAJElDCwAAQJI0tAAAACRJQwsAAECS2jU2NjaWVNiuXaX3UhHR0Jmi//F53bp1Yb7bbruFec+ePXPZ6tWrw9q99947zJcvX57LioZTvfLKK2G+Y8eOMG9NSjymFZPq+a+ksWPH5rL7778/rN1rr73CPPq6Fn2vi+5W0VC01qTa5z/Lav8OHHLIIWH+zjvv5LJvv/02rD3jjDPCvJwBY0XDZYoGOkVDzTp06FDy87Ks+HsTnZtoSFaWZdmvv/5a1jP/bdW+A7V+/stx9913h/nVV18d5kWfMWbNmpXLij6nfPTRR2He0NAQ5i0h+txVNBRt4sSJYf7XX3/lsh49eoS1W7duLX1zZar2+c+y2r8DRUOhxo8fn8seeeSRFnlmdH5racBex44dc1mXLl3KWuOBBx7IZddcc01Tt9Rk/+0OeEMLAABAkjS0AAAAJElDCwAAQJI0tAAAACRJQwsAAECSWv2U48jxxx8f5pMmTSprnWhq5dq1a8PaV199NczffvvtXLZhw4ay9tEWVHvCX2s6/5VUNGl23LhxYT5hwoRctmnTprC26N62hftS7fOfZbV/B959990wHzBgQC4r+nred999Yd6tW7cwHzJkSC7r3r17WFvJ72E0iTXLsmzRokW57JJLLglrd+7c2aJ7amnVvgO1fv5bwsiRI8N83rx5Yb777ruXvPbs2bNLXnv79u1h7ZdffhnmdXV1YX7zzTfnsqOOOiqs3bx5c5hPmzYtlz300ENhbSVV+/xnWbp3IJp+HE0+zrKWm37cmphyDAAAABWkoQUAACBJGloAAACSpKEFAAAgSRpaAAAAktQmpxyTnmpP+HP+qaZqn/8sq/07EE2dz7IsW7ZsWS478MADK7aPoq9TS3wP6+vrw/zuu+8O86Kp+ymq9h2o9fNfSf369QvzKVOm5LKhQ4c2+3nbtm0L8w8++CDM+/fvH+ZdunQp+ZkjRowI85deeqnkNSqp2uc/y1rXHSj6t+y///5hXs5fQenRo0eYjx07tuQ1in7Wr1u3ruQ1inz66adh/sorr4R5NAG/oaGh2fsolynHAAAAtEoaWgAAAJKkoQUAACBJGloAAACSZCgUSaj2QATnn2qq9vnPsnTvQJ8+fXLZfffdF9aeddZZzX7epk2bwvzOO+8M8zfeeKPktYuGPBUN0WlNqn0HUj3/lbTLLvl3Ij179gxri4a2nXvuuSU/b/DgwWG+ZcuWMF+6dGkuW7JkSVgbDY/Lstq5W9U+/1nmDlBdhkIBAADQKmloAQAASJKGFgAAgCRpaAEAAEiShhYAAIAkmXJMEqo94c/5p5qqff6zzB2guqp9B5x/qqna5z/L3AGqy5RjAAAAWiUNLQAAAEnS0AIAAJAkDS0AAABJ0tACAACQJA0tAAAASdLQAgAAkCQNLQAAAEnS0AIAAJAkDS0AAABJ0tACAACQJA0tAAAASdLQAgAAkCQNLQAAAEnS0AIAAJAkDS0AAABJ0tACAACQJA0tAAAASdLQAgAAkCQNLQAAAEnS0AIAAJAkDS0AAABJatfY2NhY7U0AAABAubyhBQAAIEkaWgAAAJKkoQUAACBJGloAAACSpKEFAAAgSRpaAAAAkqShBQAAIEkaWgAAAJKkoQUAACBJ/wNW5k2GjWiBHAAAAABJRU5ErkJggg==", "text/plain": [ "
" ] @@ -471,7 +588,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.16" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs_nnx/mnist_tutorial.md b/docs_nnx/mnist_tutorial.md index 9af0de1946..a4a05cf4ba 100644 --- a/docs_nnx/mnist_tutorial.md +++ b/docs_nnx/mnist_tutorial.md @@ -112,7 +112,7 @@ Let's put the CNN model to the test! Here, you’ll perform a forward pass with import jax.numpy as jnp # JAX NumPy y = model(jnp.ones((1, 28, 28, 1))) -y +nnx.display(y) ``` ## 4. Create the optimizer and define some metrics @@ -179,9 +179,6 @@ the accuracy) during the process. Typically this leads to the model achieving ar ```{code-cell} ipython3 :outputId: 258a2c76-2c8f-4a9e-d48b-dde57c342a87 -from IPython.display import clear_output -import matplotlib.pyplot as plt - metrics_history = { 'train_loss': [], 'train_accuracy': [], @@ -211,20 +208,40 @@ for step, batch in enumerate(train_ds.as_numpy_iterator()): metrics_history[f'test_{metric}'].append(value) metrics.reset() # Reset the metrics for the next training epoch. - clear_output(wait=True) - # Plot loss and accuracy in subplots - fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 5)) - ax1.set_title('Loss') - ax2.set_title('Accuracy') - for dataset in ('train', 'test'): - ax1.plot(metrics_history[f'{dataset}_loss'], label=f'{dataset}_loss') - ax2.plot(metrics_history[f'{dataset}_accuracy'], label=f'{dataset}_accuracy') - ax1.legend() - ax2.legend() - plt.show() + print( + f"[train] step: {step}, " + f"loss: {metrics_history['train_loss'][-1]}, " + f"accuracy: {metrics_history['train_accuracy'][-1] * 100}" + ) + print( + f"[test] step: {step}, " + f"loss: {metrics_history['test_loss'][-1]}, " + f"accuracy: {metrics_history['test_accuracy'][-1] * 100}" + ) +``` + +## 7. Visualize the metrics + +With Matplotlib, you can create plots for the loss and the accuracy: + +```{code-cell} ipython3 +:outputId: 431a2fcd-44fa-4202-f55a-906555f060ac + +import matplotlib.pyplot as plt # Visualization + +# Plot loss and accuracy in subplots +fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 5)) +ax1.set_title('Loss') +ax2.set_title('Accuracy') +for dataset in ('train', 'test'): + ax1.plot(metrics_history[f'{dataset}_loss'], label=f'{dataset}_loss') + ax2.plot(metrics_history[f'{dataset}_accuracy'], label=f'{dataset}_accuracy') +ax1.legend() +ax2.legend() +plt.show() ``` -## 7. Perform inference on the test set +## 10. Perform inference on the test set Create a `jit`-compiled model inference function (with `nnx.jit`) - `pred_step` - to generate predictions on the test set using the learned model parameters. This will enable you to visualize test images alongside their predicted labels for a qualitative assessment of model performance. diff --git a/docs_nnx/nnx_basics.ipynb b/docs_nnx/nnx_basics.ipynb index 03d0624911..351ae8b6e2 100644 --- a/docs_nnx/nnx_basics.ipynb +++ b/docs_nnx/nnx_basics.ipynb @@ -8,11 +8,7 @@ "\n", "Flax NNX is a new simplified API that is designed to make it easier to create, inspect, debug, and analyze neural networks in [JAX](https://jax.readthedocs.io/). It achieves this by adding first class support for Python reference semantics. This allows users to express their models using regular Python objects, which are modeled as PyGraphs (instead of pytrees), enabling reference sharing and mutability. Such API design should make PyTorch or Keras users feel at home.\n", "\n", - "To begin, install Flax with `pip` and import necessary dependencies:\n", - "\n", - "## Setup\n", - "\n", - "Install Flax with `pip` and impost necessary dependencies:" + "To begin, install Flax with `pip` and import necessary dependencies:" ] }, { @@ -92,7 +88,7 @@ { "data": { "text/html": [ - "
" + "
" ], "text/plain": [ "" @@ -104,7 +100,7 @@ { "data": { "text/html": [ - "
" + "
" ], "text/plain": [ "" @@ -185,18 +181,18 @@ "\n", "Flax [`nnx.Module`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/module.html)s can be used to compose other `Module`s in a nested structure. These can be assigned directly as attributes, or inside an attribute of any (nested) pytree type, such as a `list`, `dict`, `tuple`, and so on.\n", "\n", - "The example below shows how to define a simple `MLP` Module consisting of two `Linear` layers, a [`nnx.Dropout`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/nn/stochastic.html#flax.nnx.Dropout) layer, and an [`nnx.BatchNorm`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/nn/normalization.html#flax.nnx.BatchNorm) layer." + "The example below shows how to define a simple `MLP` by subclassing [`nnx.Module`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/module.html). The model consists of two `Linear` layers, an [`nnx.Dropout`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/nn/stochastic.html#flax.nnx.Dropout) layer, and an [`nnx.BatchNorm`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/nn/normalization.html#flax.nnx.BatchNorm) layer:" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "
" + "
" ], "text/plain": [ "" @@ -208,7 +204,7 @@ { "data": { "text/html": [ - "
" + "
" ], "text/plain": [ "" @@ -263,7 +259,7 @@ { "data": { "text/html": [ - "
" + "
" ], "text/plain": [ "" @@ -275,7 +271,7 @@ { "data": { "text/html": [ - "
" + "
" ], "text/plain": [ "" @@ -399,84 +395,26 @@ { "data": { "text/html": [ - "
                                              MLP Summary                                               \n",
-       "┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓\n",
-       "┃ path                 ┃ type      ┃ BatchStat           ┃ Param                ┃ RngState             ┃\n",
-       "┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩\n",
-       "│ bn                   │ BatchNorm │ mean: float32[5,32] │ bias: float32[5,32]  │                      │\n",
-       "│                      │           │ var: float32[5,32]  │ scale: float32[5,32] │                      │\n",
-       "│                      │           │                     │                      │                      │\n",
-       "│                      │           │ 320 (1.3 KB)        │ 320 (1.3 KB)         │                      │\n",
-       "├──────────────────────┼───────────┼─────────────────────┼──────────────────────┼──────────────────────┤\n",
-       "│ dropout/rngs/default │ RngStream │                     │                      │ count:               │\n",
-       "│                      │           │                     │                      │   tag: default       │\n",
-       "│                      │           │                     │                      │   value: uint32[5]   │\n",
-       "│                      │           │                     │                      │ key:                 │\n",
-       "│                      │           │                     │                      │   tag: default       │\n",
-       "│                      │           │                     │                      │   value: key<fry>[5] │\n",
-       "│                      │           │                     │                      │                      │\n",
-       "│                      │           │                     │                      │ 10 (60 B)            │\n",
-       "├──────────────────────┼───────────┼─────────────────────┼──────────────────────┼──────────────────────┤\n",
-       "│ linear1              │ Linear    │                     │ b: float32[5,32]     │                      │\n",
-       "│                      │           │                     │ w: float32[5,10,32]  │                      │\n",
-       "│                      │           │                     │                      │                      │\n",
-       "│                      │           │                     │ 1,760 (7.0 KB)       │                      │\n",
-       "├──────────────────────┼───────────┼─────────────────────┼──────────────────────┼──────────────────────┤\n",
-       "│ linear2              │ Linear    │                     │ b: float32[5,10]     │                      │\n",
-       "│                      │           │                     │ w: float32[5,32,10]  │                      │\n",
-       "│                      │           │                     │                      │                      │\n",
-       "│                      │           │                     │ 1,650 (6.6 KB)       │                      │\n",
-       "├──────────────────────┼───────────┼─────────────────────┼──────────────────────┼──────────────────────┤\n",
-       "│                      │     Total │ 320 (1.3 KB)        │ 3,730 (14.9 KB)      │ 10 (60 B)            │\n",
-       "└──────────────────────┴───────────┴─────────────────────┴──────────────────────┴──────────────────────┘\n",
-       "                                                                                                        \n",
-       "                                   Total Parameters: 4,060 (16.3 KB)                                    \n",
-       "
\n" + "
" ], "text/plain": [ - "\u001b[3m MLP Summary \u001b[0m\n", - "┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓\n", - "┃\u001b[1m \u001b[0m\u001b[1mpath \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mtype \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mBatchStat \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mParam \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mRngState \u001b[0m\u001b[1m \u001b[0m┃\n", - "┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩\n", - "│ bn │ BatchNorm │ mean: \u001b[2mfloat32\u001b[0m[5,32] │ bias: \u001b[2mfloat32\u001b[0m[5,32] │ │\n", - "│ │ │ var: \u001b[2mfloat32\u001b[0m[5,32] │ scale: \u001b[2mfloat32\u001b[0m[5,32] │ │\n", - "│ │ │ │ │ │\n", - "│ │ │ \u001b[1m320 \u001b[0m\u001b[1;2m(1.3 KB)\u001b[0m │ \u001b[1m320 \u001b[0m\u001b[1;2m(1.3 KB)\u001b[0m │ │\n", - "├──────────────────────┼───────────┼─────────────────────┼──────────────────────┼──────────────────────┤\n", - "│ dropout/rngs/default │ RngStream │ │ │ count: │\n", - "│ │ │ │ │ tag: default │\n", - "│ │ │ │ │ value: \u001b[2muint32\u001b[0m[5] │\n", - "│ │ │ │ │ key: │\n", - "│ │ │ │ │ tag: default │\n", - "│ │ │ │ │ value: \u001b[2mkey\u001b[0m[5] │\n", - "│ │ │ │ │ │\n", - "│ │ │ │ │ \u001b[1m10 \u001b[0m\u001b[1;2m(60 B)\u001b[0m │\n", - "├──────────────────────┼───────────┼─────────────────────┼──────────────────────┼──────────────────────┤\n", - "│ linear1 │ Linear │ │ b: \u001b[2mfloat32\u001b[0m[5,32] │ │\n", - "│ │ │ │ w: \u001b[2mfloat32\u001b[0m[5,10,32] │ │\n", - "│ │ │ │ │ │\n", - "│ │ │ │ \u001b[1m1,760 \u001b[0m\u001b[1;2m(7.0 KB)\u001b[0m │ │\n", - "├──────────────────────┼───────────┼─────────────────────┼──────────────────────┼──────────────────────┤\n", - "│ linear2 │ Linear │ │ b: \u001b[2mfloat32\u001b[0m[5,10] │ │\n", - "│ │ │ │ w: \u001b[2mfloat32\u001b[0m[5,32,10] │ │\n", - "│ │ │ │ │ │\n", - "│ │ │ │ \u001b[1m1,650 \u001b[0m\u001b[1;2m(6.6 KB)\u001b[0m │ │\n", - "├──────────────────────┼───────────┼─────────────────────┼──────────────────────┼──────────────────────┤\n", - "│\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1m Total\u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1m320 \u001b[0m\u001b[1;2m(1.3 KB)\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1m3,730 \u001b[0m\u001b[1;2m(14.9 KB)\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m│\u001b[1m \u001b[0m\u001b[1m10 \u001b[0m\u001b[1;2m(60 B)\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m│\n", - "└──────────────────────┴───────────┴─────────────────────┴──────────────────────┴──────────────────────┘\n", - "\u001b[1m \u001b[0m\n", - "\u001b[1m Total Parameters: 4,060 \u001b[0m\u001b[1;2m(16.3 KB)\u001b[0m\u001b[1m \u001b[0m\n" + "" ] }, "metadata": {}, "output_type": "display_data" }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n" - ] + "data": { + "text/html": [ + "
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ @@ -528,7 +466,7 @@ { "data": { "text/html": [ - "
" + "
" ], "text/plain": [ "" @@ -540,7 +478,7 @@ { "data": { "text/html": [ - "
" + "
" ], "text/plain": [ "" @@ -589,7 +527,7 @@ { "data": { "text/html": [ - "
" + "
" ], "text/plain": [ "" @@ -601,7 +539,7 @@ { "data": { "text/html": [ - "
" + "
" ], "text/plain": [ "" @@ -613,7 +551,7 @@ { "data": { "text/html": [ - "
" + "
" ], "text/plain": [ "" @@ -714,7 +652,7 @@ { "data": { "text/html": [ - "
" + "
" ], "text/plain": [ "" @@ -726,7 +664,7 @@ { "data": { "text/html": [ - "
" + "
" ], "text/plain": [ "" @@ -738,7 +676,7 @@ { "data": { "text/html": [ - "
" + "
" ], "text/plain": [ "" @@ -750,7 +688,7 @@ { "data": { "text/html": [ - "
" + "
" ], "text/plain": [ "" diff --git a/docs_nnx/nnx_basics.md b/docs_nnx/nnx_basics.md index 51e0cda53f..fbf9be0a26 100644 --- a/docs_nnx/nnx_basics.md +++ b/docs_nnx/nnx_basics.md @@ -14,10 +14,6 @@ Flax NNX is a new simplified API that is designed to make it easier to create, i To begin, install Flax with `pip` and import necessary dependencies: -## Setup - -Install Flax with `pip` and impost necessary dependencies: - ```{code-cell} ipython3 :tags: [skip-execution] @@ -95,7 +91,7 @@ to handle them, as demonstrated in later sections of this guide. Flax [`nnx.Module`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/module.html)s can be used to compose other `Module`s in a nested structure. These can be assigned directly as attributes, or inside an attribute of any (nested) pytree type, such as a `list`, `dict`, `tuple`, and so on. -The example below shows how to define a simple `MLP` Module consisting of two `Linear` layers, a [`nnx.Dropout`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/nn/stochastic.html#flax.nnx.Dropout) layer, and an [`nnx.BatchNorm`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/nn/normalization.html#flax.nnx.BatchNorm) layer. +The example below shows how to define a simple `MLP` by subclassing [`nnx.Module`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/module.html). The model consists of two `Linear` layers, an [`nnx.Dropout`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/nn/stochastic.html#flax.nnx.Dropout) layer, and an [`nnx.BatchNorm`](https://flax.readthedocs.io/en/latest/api_reference/flax.nnx/nn/normalization.html#flax.nnx.BatchNorm) layer: ```{code-cell} ipython3 class MLP(nnx.Module): diff --git a/examples/nnx_toy_examples/02_lifted_transforms.py b/examples/nnx_toy_examples/02_lifted_transforms.py index 9fef3adf26..f6d7455601 100644 --- a/examples/nnx_toy_examples/02_lifted_transforms.py +++ b/examples/nnx_toy_examples/02_lifted_transforms.py @@ -82,13 +82,15 @@ def test_step(model: MLP, batch): loss = jnp.mean((y - y_pred) ** 2) return {'loss': loss} +cached_train_step = nnx.cache_args(train_step, model, optimizer) +cached_test_step = nnx.cache_args(test_step, model) total_steps = 10_000 for step, batch in enumerate(dataset(32)): - train_step(model, optimizer, batch) + cached_train_step(batch) if step % 1000 == 0: - logs = test_step(model, (X, Y)) + logs = cached_test_step((X, Y)) print(f"step: {step}, loss: {logs['loss']}") if step >= total_steps - 1: diff --git a/flax/configurations.py b/flax/configurations.py index ba19a572fc..5e1a492fcf 100644 --- a/flax/configurations.py +++ b/flax/configurations.py @@ -22,6 +22,7 @@ class Config: + flax_use_flaxlib: bool # See https://google.github.io/pytype/faq.html. _HAS_DYNAMIC_ATTRIBUTES = True @@ -62,6 +63,10 @@ def update(self, name_or_holder, value, /): raise LookupError(f'Unrecognized config option: {name}') self._values[name] = value + def __repr__(self): + values_repr = ', '.join(f'\n {k}={v!r}' for k, v in self._values.items()) + return f'Config({values_repr}\n)' + config = Config() @@ -201,3 +206,9 @@ def temp_flip_flag(var_name: str, var_value: bool): ' PRNG keys.' ), ) + +flax_use_flaxlib = bool_flag( + name='flax_use_flaxlib', + default=False, + help='Whether to use flaxlib for C++ acceleration.', +) \ No newline at end of file diff --git a/flax/linen/module.py b/flax/linen/module.py index 52e5a0594b..f8a57b9546 100644 --- a/flax/linen/module.py +++ b/flax/linen/module.py @@ -1274,11 +1274,6 @@ def __setattr__(self, name: str, val: Any): object.__setattr__(self, name, val) return else: - # If the attribute is a python special method, we allow setting it (this - # is useful e.g. for IPython auto-reload). - if name.startswith('__'): - object.__setattr__(self, name, val) - return # We're past all initialization and setup logic: # Raises a TypeError just like frozen python dataclasses. raise errors.SetAttributeFrozenModuleError( diff --git a/flax/nnx/__init__.py b/flax/nnx/__init__.py index fcb15f0608..1c0c19a46f 100644 --- a/flax/nnx/__init__.py +++ b/flax/nnx/__init__.py @@ -56,6 +56,7 @@ from .graph import MergeContext as MergeContext from .graph import merge_context as merge_context from .graph import variables as variables +from .graph import cache_args as cache_args from .nn import initializers as initializers from .nn.activations import celu as celu from .nn.activations import elu as elu diff --git a/flax/nnx/bridge/variables.py b/flax/nnx/bridge/variables.py index 121bb98eb8..b1b78d1684 100644 --- a/flax/nnx/bridge/variables.py +++ b/flax/nnx/bridge/variables.py @@ -18,10 +18,9 @@ import jax from flax import struct from flax.core import meta -from flax.nnx import spmd +from flax.nnx import graph, spmd from flax.nnx import traversals from flax.nnx import variablelib as variableslib -from flax.nnx.module import GraphDef import typing as tp @@ -174,7 +173,6 @@ def _recursive_merge(dict1, dict2): def linen_vars_to_nnx_attrs(variables: tp.Mapping[str, Any]) -> dict[str, Any]: - """Convert a dict of Linen-style variables to NNX variables.""" nnx_vars = jax.tree_util.tree_map_with_path( lambda kp, x: to_nnx_var(get_col_name(kp), x), variables, is_leaf=lambda x: isinstance(x, meta.AxisMetadata)) @@ -191,22 +189,21 @@ def linen_vars_to_nnx_attrs(variables: tp.Mapping[str, Any]) -> dict[str, Any]: def nnx_attrs_to_linen_vars(nnx_attrs: dict) -> dict: - """Convert a dict of NNX variables (or variable states) to Linen-style variables.""" linen_structured = {} for kp, v in traversals.flatten_mapping( - nnx_attrs, - is_leaf=lambda _, x: isinstance( - x, variableslib.Variable | variableslib.VariableState | GraphDef - ), + nnx_attrs, + is_leaf=lambda _, x: isinstance( + x, variableslib.Variable | graph.NodeDef | graph.NodeRef + ), ).items(): if isinstance(v, variableslib.Variable): col_name = variable_type_name(type(v)) - v = to_linen_var(v.to_state()) - elif isinstance(v, variableslib.VariableState): - col_name = variable_type_name(v.type) - v = to_linen_var(v) else: col_name = 'nnx' # it must be an nnx.GraphDef, for some ToLinen submodule linen_structured[(col_name, *kp)] = v variables = traversals.unflatten_mapping(linen_structured) + variables = jax.tree.map(lambda x: to_linen_var(x.to_state()), + variables, + is_leaf=lambda x: isinstance(x, variableslib.Variable)) return variables + diff --git a/flax/nnx/extract.py b/flax/nnx/extract.py index 191a0c195a..364177b5f5 100644 --- a/flax/nnx/extract.py +++ b/flax/nnx/extract.py @@ -13,9 +13,6 @@ # limitations under the License. import abc -import contextlib -import dataclasses -import threading import typing as tp import jax @@ -67,7 +64,7 @@ def extract_graph_nodes( | tuple[A, tuple[tp.Any, ...], tuple[tp.Any, ...]] ): """Extracts all graph nodes from a pytree.""" - nodes = graph.RefMap[tp.Any, Index]() + nodes: dict[tp.Any, Index] = {} node_prefixes = [] leaves = [] @@ -134,11 +131,10 @@ def check_consistent_aliasing( prefix: tuple[tp.Any, ...], /, *, - node_prefixes: graph.RefMap[tp.Any, list[tuple[PathParts, tp.Any]]] - | None = None, + node_prefixes: dict[tp.Any, list[tuple[PathParts, tp.Any]]] | None = None, ): if node_prefixes is None: - node_prefixes = graph.RefMap() + node_prefixes = {} # collect all paths and prefixes for each node for path, value in graph.iter_graph(node): @@ -181,50 +177,6 @@ def check_consistent_aliasing( + '\n'.join(node_msgs) ) - -# ----------------------------- -# broadcast -# ----------------------------- - - -@dataclasses.dataclass -class BroadcastContext(threading.local): - broadcast_state_stacks: dict[str, list[tp.Any]] = dataclasses.field( - default_factory=dict - ) - - -BROADCAST_CONTEXT = BroadcastContext() - - -@contextlib.contextmanager -def broadcast_state(tag: str, state: tp.Any): - if tag in BROADCAST_CONTEXT.broadcast_state_stacks: - stack = BROADCAST_CONTEXT.broadcast_state_stacks[tag] - else: - stack = BROADCAST_CONTEXT.broadcast_state_stacks[tag] = [] - stack.append(state) - try: - yield - finally: - stack.pop() - if not stack: - del BROADCAST_CONTEXT.broadcast_state_stacks[tag] - - -def get_broadcast_state(tag: str) -> tp.Any: - if tag not in BROADCAST_CONTEXT.broadcast_state_stacks: - raise ValueError(f'No broadcast state found for {tag!r}') - - stack = BROADCAST_CONTEXT.broadcast_state_stacks[tag] - - if not stack: - raise RuntimeError( - f'Empty broadcast state stack for {tag!r}, this is a bug' - ) - - return stack[-1] - # ----------------------------- # to_tree/from_tree # ----------------------------- @@ -251,10 +203,13 @@ class GraphDefState(struct.PyTreeNode): graphdef: graph.GraphDef[tp.Any] = struct.field(pytree_node=False) state: graph.GraphState = struct.field(pytree_node=True) +S = tp.TypeVar( + 'S', bound=graph.GraphState | graph.GraphFlatState | list[tp.Any] +) -class NodeStates(struct.PyTreeNode): +class NodeStates(struct.PyTreeNode, tp.Generic[S]): _graphdef: graph.GraphDef[tp.Any] | None - states: tuple[graph.GraphState, ...] + states: tuple[S, ...] metadata: tp.Any = struct.field(pytree_node=False) @property @@ -264,7 +219,7 @@ def graphdef(self) -> graph.GraphDef[tp.Any]: return self._graphdef @property - def state(self) -> graph.GraphState: + def state(self) -> S: if len(self.states) != 1: raise ValueError( f'Expected exactly one GraphDefState, got {len(self.states)}' @@ -275,15 +230,19 @@ def state(self) -> graph.GraphState: def from_split( cls, graphdef: graph.GraphDef[tp.Any], - state: graph.GraphState, + state: S, /, - *states: graph.GraphState, + *states: S, metadata: tp.Any = None, ): return cls(_graphdef=graphdef, states=(state, *states), metadata=metadata) @classmethod - def from_states(cls, state: graph.GraphState, *states: graph.GraphState): + def from_states( + cls, + state: S, + *states: S, + ): return cls(_graphdef=None, states=(state, *states), metadata=None) @classmethod @@ -312,9 +271,18 @@ def to_tree( [graph.SplitContext, KeyPath, Prefix, Leaf], tp.Any ] = default_split_fn, map_non_graph_nodes: bool = False, - ctxtag: str | None = None, + ctxtag: tp.Hashable | None = None, check_aliasing: bool = True, ) -> tp.Any: + if prefix is Missing or prefix is None: + # fast path, no need for prefix broadcasting or consistent aliasing checks + with graph.split_context(ctxtag) as split_ctx: + return jax.tree.map( + lambda x: split_fn(split_ctx, (), prefix, x) + if map_non_graph_nodes or graph.is_graph_node(x) + else x, + tree, + ) leaf_prefixes = broadcast_prefix( prefix, tree, @@ -324,7 +292,7 @@ def to_tree( assert len(leaf_keys) == len(leaf_prefixes) leaves_out = [] - node_prefixes = graph.RefMap[tp.Any, list[tuple[PathParts, tp.Any]]]() + node_prefixes: dict[tp.Any, list[tuple[PathParts, tp.Any]]] = {} with graph.split_context(ctxtag) as split_ctx: for (keypath, leaf), leaf_prefix in zip(leaf_keys, leaf_prefixes): @@ -367,8 +335,19 @@ def from_tree( is_node_leaf: tp.Callable[[Leaf], bool] = is_tree_node, is_leaf: tp.Callable[[Leaf], bool] = is_tree_node, map_non_graph_nodes: bool = False, - ctxtag: str | None = None, + is_inner: bool | None = None, + ctxtag: tp.Hashable | None = None, ) -> tp.Any: + if prefix is Missing or prefix is None: + # fast path, no need for prefix broadcasting or consistent aliasing checks + with graph.merge_context(is_inner, ctxtag) as merge_ctx: + return jax.tree.map( + lambda x: merge_fn(merge_ctx, (), prefix, x) + if map_non_graph_nodes or is_node_leaf(x) + else x, + tree, + is_leaf=is_leaf, + ) leaf_prefixes = broadcast_prefix( prefix, tree, @@ -381,15 +360,11 @@ def from_tree( assert len(leaf_keys) == len(leaf_prefixes) leaves_out = [] - with graph.merge_context(ctxtag) as merge_ctx: + with graph.merge_context(is_inner, ctxtag) as merge_ctx: for (keypath, leaf), leaf_prefix in zip(leaf_keys, leaf_prefixes): - if is_node_leaf(leaf): - leaf_out = merge_fn(merge_ctx, keypath, leaf_prefix, leaf) - leaves_out.append(leaf_out) - else: - if map_non_graph_nodes: - leaf = merge_fn(merge_ctx, keypath, leaf_prefix, leaf) - leaves_out.append(leaf) + if map_non_graph_nodes or is_node_leaf(leaf): + leaf = merge_fn(merge_ctx, keypath, leaf_prefix, leaf) + leaves_out.append(leaf) pytree_out = jax.tree.unflatten(treedef, leaves_out) return pytree_out diff --git a/flax/nnx/filterlib.py b/flax/nnx/filterlib.py index 1028efb2b1..63ed371be9 100644 --- a/flax/nnx/filterlib.py +++ b/flax/nnx/filterlib.py @@ -54,9 +54,7 @@ def to_predicate(filter: Filter) -> Predicate: else: raise TypeError(f'Invalid collection filter: {filter:!r}. ') -def filters_to_predicates( - filters: tp.Sequence[Filter], -) -> tuple[Predicate, ...]: +def filters_to_predicates(filters: tuple[Filter, ...]) -> tuple[Predicate, ...]: for i, filter_ in enumerate(filters): if filter_ in (..., True) and i != len(filters) - 1: remaining_filters = filters[i + 1 :] diff --git a/flax/nnx/graph.py b/flax/nnx/graph.py index 8cc272f8eb..8caf7e8a8c 100644 --- a/flax/nnx/graph.py +++ b/flax/nnx/graph.py @@ -14,23 +14,26 @@ from __future__ import annotations +from collections import deque import contextlib import dataclasses import functools import threading import typing as tp +from weakref import WeakKeyDictionary +from flax import config import jax import numpy as np import typing_extensions as tpe -from flax.nnx import filterlib, reprlib, visualization +from flax.nnx import filterlib, reprlib from flax.nnx.proxy_caller import ( ApplyCaller, CallableProxy, DelayedAccessor, ) -from flax.nnx.statelib import State +from flax.nnx.statelib import FlatState, State from flax.nnx import variablelib from flax.nnx.variablelib import Variable, VariableState from flax.typing import Key, PathParts, is_key_like @@ -53,6 +56,7 @@ StateLeaf = VariableState[tp.Any] NodeLeaf = Variable[tp.Any] GraphState = State[Key, StateLeaf] +GraphFlatState = FlatState[StateLeaf] def is_state_leaf(x: tp.Any) -> tpe.TypeGuard[StateLeaf]: @@ -62,37 +66,12 @@ def is_state_leaf(x: tp.Any) -> tpe.TypeGuard[StateLeaf]: def is_node_leaf(x: tp.Any) -> tpe.TypeGuard[NodeLeaf]: return isinstance(x, Variable) +RefMap = dict -class RefMap(tp.MutableMapping[A, B], reprlib.MappingReprMixin): - """A mapping that uses object id as the hash for the keys.""" - - def __init__( - self, mapping: tp.Mapping[A, B] | tp.Iterable[tuple[A, B]] = (), / - ): - self._mapping: dict[int, tuple[A, B]] = {} - self.update(mapping) - - def __getitem__(self, key: A) -> B: - return self._mapping[id(key)][1] - - def __contains__(self, key: object) -> bool: - return id(key) in self._mapping - - def __setitem__(self, key: A, value: B): - self._mapping[id(key)] = (key, value) - - def __delitem__(self, key: A): - del self._mapping[id(key)] - - def __iter__(self) -> tp.Iterator[A]: - return (key for key, _ in self._mapping.values()) - - def __len__(self) -> int: - return len(self._mapping) - - def __str__(self) -> str: - return repr(self) +if not tp.TYPE_CHECKING and config.flax_use_flaxlib: + import flaxlib + RefMap = flaxlib.RefMap @dataclasses.dataclass(frozen=True, slots=True) class NodeImplBase(tp.Generic[Node, Leaf, AuxData]): @@ -175,9 +154,9 @@ def is_node_type(x: type[tp.Any]) -> bool: return x in GRAPH_REGISTRY or x in PYTREE_REGISTRY or x is GenericPytree -def get_node_impl(x: Node) -> NodeImpl[Node, tp.Any, tp.Any]: +def get_node_impl(x: Node) -> NodeImpl[Node, tp.Any, tp.Any] | None: if isinstance(x, Variable): - raise ValueError(f'Variable is not a node: {x}') + return None node_type = type(x) @@ -185,19 +164,23 @@ def get_node_impl(x: Node) -> NodeImpl[Node, tp.Any, tp.Any]: return GRAPH_REGISTRY[node_type] elif node_type in PYTREE_REGISTRY: return PYTREE_REGISTRY[node_type] - elif is_pytree_node(x): + elif node_type in JAX_PYTREE_REGISTRY or issubclass(node_type, tuple): return PYTREE_NODE_IMPL # type: ignore else: - raise ValueError(f'Unknown node type: {x}') + return None -def get_node_impl_for_type(x: type[Node]) -> NodeImpl[Node, tp.Any, tp.Any]: +def get_node_impl_for_type( + x: type[Node], +) -> NodeImpl[Node, tp.Any, tp.Any] | None: if x is GenericPytree: return PYTREE_NODE_IMPL # type: ignore elif x in PYTREE_REGISTRY: return PYTREE_REGISTRY[x] - else: + elif x in GRAPH_REGISTRY: return GRAPH_REGISTRY[x] + else: + return None class HashableMapping(tp.Mapping[HA, HB], tp.Hashable): @@ -228,17 +211,8 @@ def __repr__(self) -> str: return repr(self._mapping) -class GraphDef(tp.Generic[Node]): - """A class that represents all the static, stateless, and Pythonic parts of a Flax - :class:`Module`. A ``GraphDef`` can be generated by either calling :func:`split` or - :func:`graphdef` on the :class:`Module`.""" - - type: type[Node] - index: int - - @dataclasses.dataclass(frozen=True, repr=False) -class NodeRef(GraphDef[Node], reprlib.Representable): +class NodeRef(tp.Generic[Node], reprlib.Representable): type: type[Node] index: int @@ -248,7 +222,8 @@ def __nnx_repr__(self): yield reprlib.Attr('index', self.index) def __treescope_repr__(self, path, subtree_renderer): - return visualization.render_object_constructor( + import treescope # type: ignore[import-not-found,import-untyped] + return treescope.repr_lib.render_object_constructor( object_type=type(self), attributes={'type': self.type, 'index': self.index}, path=path, @@ -262,16 +237,33 @@ def __treescope_repr__(self, path, subtree_renderer): class VariableDef(reprlib.Representable): type: type[Variable] index: int + outer_index: int | None metadata: HashableMapping[str, tp.Any] + def with_no_outer_index(self) -> VariableDef: + return VariableDef( + type=self.type, index=self.index, outer_index=None, metadata=self.metadata + ) + + def with_same_outer_index(self) -> VariableDef: + return VariableDef( + type=self.type, + index=self.index, + outer_index=self.index, + metadata=self.metadata, + ) + def __nnx_repr__(self): yield reprlib.Object(type=type(self)) yield reprlib.Attr('type', self.type.__name__) yield reprlib.Attr('index', self.index) + yield reprlib.Attr('outer_index', self.outer_index) yield reprlib.Attr('metadata', reprlib.PrettyMapping(self.metadata)) def __treescope_repr__(self, path, subtree_renderer): - return visualization.render_object_constructor( + import treescope # type: ignore[import-not-found,import-untyped] + + return treescope.repr_lib.render_object_constructor( object_type=type(self), attributes={ 'type': self.type, @@ -286,71 +278,74 @@ def __treescope_repr__(self, path, subtree_renderer): jax.tree_util.register_static(VariableDef) -@dataclasses.dataclass(frozen=True, slots=True) -class SubGraphAttribute: - key: Key - value: NodeDef[tp.Any] | NodeRef[tp.Any] - - -@dataclasses.dataclass(frozen=True, slots=True) -class StaticAttribute: - key: Key - value: tp.Any - - -@dataclasses.dataclass(frozen=True, slots=True) -class LeafAttribute: - key: Key - value: VariableDef | NodeRef[tp.Any] - - @dataclasses.dataclass(frozen=True, repr=False, slots=True) -class NodeDef(GraphDef[Node], reprlib.Representable): +class NodeDef(tp.Generic[Node], reprlib.Representable): """A dataclass that denotes the tree structure of a :class:`Module`. A ``GraphDef`` can be generated by either calling :func:`split` or :func:`graphdef` on the :class:`Module`.""" type: tp.Type[Node] index: int - attributes: tuple[SubGraphAttribute | StaticAttribute | LeafAttribute, ...] + outer_index: int | None + attributes: tuple[ + tuple[ + Key, NodeDef[tp.Any] | VariableDef | NodeRef[tp.Any] | Static[tp.Any] + ], + ..., + ] metadata: tp.Any - index_mapping: HashableMapping[Index, Index] | None - @classmethod - def create( - cls, - type: tp.Type[Node], - index: int, - attributes: tuple[SubGraphAttribute | StaticAttribute | LeafAttribute, ...], - metadata: tp.Any, - index_mapping: tp.Mapping[Index, Index] | None, - ): - return cls( - type=type, - index=index, + def with_no_outer_index(self) -> NodeDef[Node]: + attributes = tuple( + ( + key, + value.with_no_outer_index() + if isinstance(value, NodeDef | VariableDef) + else value, + ) + for key, value in self.attributes + ) + return NodeDef( + type=self.type, + index=self.index, + outer_index=None, attributes=attributes, - metadata=metadata, - index_mapping=HashableMapping(index_mapping) - if index_mapping is not None - else None, + metadata=self.metadata, ) + def with_same_outer_index(self) -> NodeDef[Node]: + attributes = tuple( + ( + key, + value.with_same_outer_index() + if isinstance(value, NodeDef | VariableDef) + else value, + ) + for key, value in self.attributes + ) + return NodeDef( + type=self.type, + index=self.index, + outer_index=self.index if self.index >= 0 else None, + attributes=attributes, + metadata=self.metadata, + ) + + def replace(self, **kwargs): + return dataclasses.replace(self, **kwargs) + def __nnx_repr__(self): yield reprlib.Object(type=type(self)) yield reprlib.Attr('type', self.type.__name__) yield reprlib.Attr('index', self.index) - yield reprlib.Attr('attributes', reprlib.PrettySequence(self.attributes)) + yield reprlib.Attr('outer_index', self.outer_index) + yield reprlib.Attr('attributes', self.attributes) yield reprlib.Attr('metadata', self.metadata) - yield reprlib.Attr( - 'index_mapping', - reprlib.PrettyMapping(self.index_mapping) - if self.index_mapping is not None - else None, - ) def __treescope_repr__(self, path, subtree_renderer): - return visualization.render_object_constructor( + import treescope # type: ignore[import-not-found,import-untyped] + return treescope.repr_lib.render_object_constructor( object_type=type(self), attributes={ 'type': self.type, @@ -373,19 +368,89 @@ def _apply( module = merge(self, state, *states) fn = accessor(module) out = fn(*args, **kwargs) - return out, flatten(module) + graphdef, flat_state = flatten(module) + state_ = State.from_flat_path(flat_state) + return out, (graphdef, state_) return CallableProxy(_apply, accessor) # type: ignore jax.tree_util.register_static(NodeDef) -PureState = tuple[GraphDef[A], GraphState] +GraphDef = tp.Union[NodeDef[Node], NodeRef[Node]] +PureState = tuple[GraphDef[Node], GraphState] +@tp.overload def flatten( - node: Node, /, ref_index: RefMap[tp.Any, Index] | None = None -) -> tuple[GraphDef[Node], GraphState]: + node: Node, + /, + *, + ref_index: RefMap | None = None, + ref_outer_index: RefMap | None = None, +) -> tuple[GraphDef[Node], FlatState[VariableState[tp.Any]]]: ... +@tp.overload +def flatten( + node: Node, + /, + *, + with_paths: tp.Literal[True], + return_variables: tp.Literal[True], + ref_index: RefMap | None = None, + ref_outer_index: RefMap | None = None, +) -> tuple[ + GraphDef[Node], + FlatState[Variable[tp.Any]], +]: ... +@tp.overload +def flatten( + node: Node, + /, + *, + with_paths: tp.Literal[False], + return_variables: tp.Literal[True], + ref_index: RefMap | None = None, + ref_outer_index: RefMap | None = None, +) -> tuple[ + GraphDef[Node], + list[Variable[tp.Any]], +]: ... +@tp.overload +def flatten( + node: Node, + /, + *, + return_variables: tp.Literal[True], + ref_index: RefMap | None = None, + ref_outer_index: RefMap | None = None, +) -> tuple[ + GraphDef[Node], + FlatState[Variable[tp.Any]], +]: ... +@tp.overload +def flatten( + node: Node, + /, + *, + with_paths: bool, + ref_index: RefMap | None = None, + ref_outer_index: RefMap | None = None, +) -> tuple[ + GraphDef[Node], + FlatState[VariableState[tp.Any]] | list[tp.Any], +]: ... +def flatten( + node: Node, + /, + *, + with_paths: bool = True, + return_variables: bool = False, + ref_index: RefMap | None = None, + ref_outer_index: RefMap | None = None, +) -> tuple[ + GraphDef[Node], + FlatState[VariableState[tp.Any]] | FlatState[Variable[tp.Any]] | list[tp.Any], +]: """Flattens a graph node into a (graphdef, state) pair. Args: @@ -393,81 +458,355 @@ def flatten( ref_index: A mapping from nodes to indexes, defaults to None. If not provided, a new empty dictionary is created. This argument can be used to flatten a sequence of graph nodes that share references. + with_paths: A boolean that indicates whether to return a FlatState object that includes + the paths to VariableState objects, or just a list of the Variable's inner values. """ if ref_index is None: ref_index = RefMap() - flat_state: list[tuple[PathParts, StateLeaf]] = [] - graphdef = _graph_flatten((), ref_index, flat_state, node) - return graphdef, GraphState.from_flat_path(flat_state) + + leaves: list[StateLeaf | Variable[tp.Any]] = [] + path: list[Key] | None = [] if with_paths else None + paths: list[PathParts] | None = [] if with_paths else None + node_impl = get_node_impl(node) + if node_impl is None: + raise RuntimeError(f'Unsupported type: {type(node)}, this is a bug.') + graphdef = _graph_flatten( + node, + node_impl, + path, + ref_index, + ref_outer_index, + leaves, + paths, + return_variables, + ) + + if paths is not None: + return graphdef, FlatState.from_sorted_keys_values(tuple(paths), leaves) + else: + return graphdef, leaves def _graph_flatten( - path: PathParts, - ref_index: RefMap[tp.Any, Index], - flat_state: list[tuple[PathParts, StateLeaf]], node: Node, + node_impl: NodeImpl[Node, Leaf, AuxData], + path: list[Key] | None, + ref_index: RefMap, + ref_outer_index: RefMap | None, + leaves: list[StateLeaf | Variable[tp.Any]], + paths: list[PathParts] | None, + return_variables: bool, ) -> NodeDef[Node] | NodeRef: - if not is_node(node): - raise RuntimeError(f'Unsupported type: {type(node)}, this is a bug.') + is_pytree_node_ = isinstance(node_impl, PytreeNodeImpl) + is_graph_node_ = isinstance(node_impl, GraphNodeImpl) - if node in ref_index: + if not is_pytree_node_ and node in ref_index: return NodeRef(type(node), ref_index[node]) - node_impl = get_node_impl(node) - # only cache graph nodes - if isinstance(node_impl, GraphNodeImpl): + if is_graph_node_: index = len(ref_index) ref_index[node] = index else: index = -1 - attributes: list[SubGraphAttribute | StaticAttribute | LeafAttribute] = [] + attributes: list[ + tuple[Key, Static[tp.Any] | NodeDef[tp.Any] | VariableDef | NodeRef[tp.Any]] + ] = [] values, metadata = node_impl.flatten(node) for key, value in values: - if is_node(value): - nodedef = _graph_flatten((*path, key), ref_index, flat_state, value) - # subgraphs.append((key, nodedef)) - attributes.append(SubGraphAttribute(key, nodedef)) + value_node_impl = get_node_impl(value) + if path is not None: + path.append(key) + if value_node_impl is not None: + nodedef = _graph_flatten( + value, + value_node_impl, + path, + ref_index, + ref_outer_index, + leaves, + paths, + return_variables, + ) + attributes.append((key, nodedef)) elif isinstance(value, Variable): if value in ref_index: - attributes.append( - LeafAttribute(key, NodeRef(type(value), ref_index[value])) - ) + attributes.append((key, NodeRef(type(value), ref_index[value]))) else: - flat_state.append(((*path, key), value.to_state())) + if return_variables: + leaf = value + elif path is None: + leaf = value.raw_value + else: + leaf = value.to_state() + leaves.append(leaf) + if path is not None: + assert paths is not None + paths.append(tuple(path)) variable_index = ref_index[value] = len(ref_index) variabledef = VariableDef( - type(value), variable_index, HashableMapping(value._var_metadata) + type=type(value), + index=variable_index, + outer_index=ref_outer_index.get(value, None) + if ref_outer_index + else None, + metadata=HashableMapping(value._var_metadata), ) - attributes.append(LeafAttribute(key, variabledef)) + attributes.append((key, variabledef)) else: if isinstance(value, (jax.Array, np.ndarray)): - path_str = '/'.join(map(str, (*path, key))) - raise ValueError( + if path is not None: + path_str = '/'.join(map(str, path)) + raise ValueError( f'Arrays leaves are not supported, at {path_str!r}: {value}' - ) + ) + else: + raise ValueError(f'Arrays leaves are not supported, found {value}') # static_fields.append((key, value)) - attributes.append(StaticAttribute(key, value)) + attributes.append((key, Static(value))) - nodedef = NodeDef.create( + if path is not None: + path.pop() + + nodedef = NodeDef( type=node_impl.type, index=index, + outer_index=ref_outer_index[node] + if is_graph_node_ and ref_outer_index and node in ref_outer_index + else None, attributes=tuple(attributes), metadata=metadata, - index_mapping=None, ) return nodedef +@dataclasses.dataclass(slots=True) +class FingerprintContext: + next_index: int + +def fingerprint( + node, + /, + *, + ref_index: RefMap | None = None, + new_ref_index: RefMap | None = None, +) -> list[tp.Hashable]: + """ """ + if ref_index is None: + ref_index = RefMap() + + if new_ref_index is None: + new_ref_index = RefMap() + node_impl = get_node_impl(node) + if node_impl is None: + raise RuntimeError(f'Unsupported type: {type(node)}, this is a bug.') + ctx = FingerprintContext(len(ref_index) + len(new_ref_index)) + fp: list[tp.Hashable] = [] + _graph_fingerprint(ctx, fp.append, node, node_impl, ref_index, new_ref_index) + return fp + + +def _graph_fingerprint( + ctx: FingerprintContext, + append_fn: tp.Callable[[tp.Hashable], None], + node, + node_impl: NodeImpl[Node, Leaf, AuxData], + ref_index: RefMap, + new_ref_index: RefMap, +): + is_pytree_node_ = type(node_impl) is PytreeNodeImpl + is_graph_node_ = type(node_impl) is GraphNodeImpl + + append_fn(type(node)) + + if is_graph_node_: + append_fn(id(node)) + if node in ref_index: + append_fn(ref_index[node]) + return + elif node in new_ref_index: + append_fn(new_ref_index[node]) + return + index = new_ref_index[node] = ctx.next_index + ctx.next_index += 1 + else: + index = -1 + + values, metadata = node_impl.flatten(node) + + append_fn(index) + append_fn(metadata) + + for key, value in values: + value_node_impl = get_node_impl(value) + append_fn(key) + if value_node_impl is not None: + _graph_fingerprint( + ctx, + append_fn, + value, + value_node_impl, + ref_index, + new_ref_index, + ) + elif isinstance(value, Variable): + append_fn(id(value)) + append_fn(type(value)) + if value in ref_index: + append_fn(ref_index[value]) + elif value in new_ref_index: + append_fn(new_ref_index[value]) + else: + variable_index = new_ref_index[value] = ctx.next_index + ctx.next_index += 1 + append_fn(variable_index) + for key_value in value._var_metadata.items(): + append_fn(key_value) + else: + if isinstance(value, (jax.Array, np.ndarray)): + raise ValueError(f'Arrays leaves are not supported: {value}') + append_fn(value) + +def check_fingerprint( + node, + fp: list[tp.Hashable], + /, + *, + ref_index: RefMap | None = None, + new_ref_index: RefMap | None = None, +) -> bool: + """ """ + if ref_index is None: + ref_index = RefMap() + + if new_ref_index is None: + new_ref_index = RefMap() + node_impl = get_node_impl(node) + if node_impl is None: + raise RuntimeError(f'Unsupported type: {type(node)}, this is a bug.') + ctx = FingerprintContext(len(ref_index) + len(new_ref_index)) + fp_matches = _check_graph_fingerprint( + ctx, iter(fp), node, node_impl, ref_index, new_ref_index + ) + return fp_matches + + +def _check_graph_fingerprint( + ctx: FingerprintContext, + fp_iterator: tp.Iterator[tp.Hashable], + node, + node_impl: NodeImpl[Node, Leaf, AuxData], + ref_index: RefMap, + new_ref_index: RefMap, +) -> bool: + is_pytree_node_ = type(node_impl) is PytreeNodeImpl + is_graph_node_ = type(node_impl) is GraphNodeImpl + + if type(node) != next(fp_iterator): + return False + + if is_graph_node_: + # append_fn(id(node)) + if id(node) != next(fp_iterator): + return False + if node in ref_index: + # append_fn(ref_index[node]) + return ref_index[node] == next(fp_iterator) + elif node in new_ref_index: + # append_fn(new_ref_index[node]) + return new_ref_index[node] == next(fp_iterator) + index = new_ref_index[node] = ctx.next_index + ctx.next_index += 1 + else: + index = -1 + + values, metadata = node_impl.flatten(node) + + # append_fn(index) + if index != next(fp_iterator): + return False + # append_fn(metadata) + if metadata != next(fp_iterator): + return False + + for key, value in values: + value_node_impl = get_node_impl(value) + # append_fn(key) + if key != next(fp_iterator): + return False + if value_node_impl is not None: + if not _check_graph_fingerprint( + ctx, + fp_iterator, + value, + value_node_impl, + ref_index, + new_ref_index, + ): + return False + elif isinstance(value, Variable): + # append_fn(id(value)) + if id(value) != next(fp_iterator): + return False + # append_fn(type(value)) + if type(value) != next(fp_iterator): + return False + if value in ref_index: + # append_fn(ref_index[value]) + if ref_index[value] != next(fp_iterator): + return False + elif value in new_ref_index: + # append_fn(new_ref_index[value]) + if new_ref_index[value] != next(fp_iterator): + return False + else: + variable_index = new_ref_index[value] = ctx.next_index + ctx.next_index += 1 + # append_fn(variable_index) + if variable_index != next(fp_iterator): + return False + for key_value in value._var_metadata.items(): + # append_fn(key_value) + if key_value != next(fp_iterator): + return False + else: + if isinstance(value, (jax.Array, np.ndarray)): + raise ValueError(f'Arrays leaves are not supported: {value}') + # append_fn(value) + if value != next(fp_iterator): + return False + + return True + + +def _get_sorted_leaves( + xs: tp.Mapping[tp.Any, tp.Any], +) -> deque[tp.Any]: + if not isinstance(xs, tp.Mapping): # type: ignore + raise TypeError(f'expected Mapping; got {type(xs).__qualname__}') + leaves = deque() + + def _flatten(xs): + if not isinstance(xs, tp.Mapping): + leaves.append(xs) + else: + for _, value in sorted(xs.items()): + _flatten(value) + + _flatten(xs) + return leaves + def unflatten( graphdef: GraphDef[Node], - state: tp.Mapping[KeyT, StateLeaf | tp.Mapping[Key, tp.Any]], + state: State[KeyT, tp.Any | dict[KeyT, tp.Any]] + | FlatState[tp.Any] + | list[tp.Any], /, *, index_ref: dict[Index, tp.Any] | None = None, - index_ref_cache: dict[Index, tp.Any] | None = None, + outer_index_outer_ref: dict[Index, tp.Any] | None = None, ) -> Node: """Unflattens a graphdef into a node with the given state. @@ -484,19 +823,41 @@ def unflatten( existing graph nodes are mutated to have the new content/topology specified by the graphdef. """ - if isinstance(state, State): - state = state.raw_mapping # type: ignore + if isinstance(state, (State, dict)): + leaves = _get_sorted_leaves(state) + elif isinstance(state, FlatState): + leaves = deque(state.get_values()) + elif isinstance(state, list): # type: ignore + leaves = deque(state) + else: + raise ValueError(f'Unsupported state type: {type(state)}') if index_ref is None: index_ref = {} - assert isinstance(graphdef, (NodeDef, NodeRef)) - node = _graph_unflatten(graphdef, state, index_ref, index_ref_cache) + + if isinstance(graphdef, NodeRef): + node = index_ref[graphdef.index] + else: + assert isinstance(graphdef, NodeDef) + node_impl = get_node_impl_for_type(graphdef.type) + if node_impl is None: + raise RuntimeError(f'Unsupported type: {graphdef.type}, this is a bug.') + node = _graph_unflatten( + graphdef, node_impl, leaves, index_ref, outer_index_outer_ref + ) + if leaves: + raise ValueError( + f'Incorrect number of leaves: got an extra {len(leaves)} leaves in the state' + ) + return node + def _graph_unflatten( nodedef: NodeDef[Node] | NodeRef[Node], - state: tp.Mapping[KeyT, StateLeaf | tp.Mapping[Key, tp.Any]], + node_impl: NodeImpl[Node, Leaf, AuxData], + leaves: deque[tp.Any], index_ref: dict[Index, tp.Any], - index_ref_cache: dict[Index, tp.Any] | None, + outer_index_outer_ref: dict[Index, tp.Any] | None, ) -> Node: """Recursive helper for graph_unflatten. @@ -511,134 +872,82 @@ def _graph_unflatten( existing graph nodes are mutated to have the new content/topology specified by the nodedef. """ - if isinstance(nodedef, NodeRef): + if type(nodedef) is NodeRef: return index_ref[nodedef.index] - if not is_node_type(nodedef.type): - raise RuntimeError(f'Unsupported type: {nodedef.type}, this is a bug.') - if nodedef.index in index_ref: raise RuntimeError(f'GraphDef index {nodedef.index} already used.') - node_impl = get_node_impl_for_type(nodedef.type) - def _get_children(): children: list[tuple[Key, NodeLeaf | Node]] = [] - state_keys: set = set(state.keys()) - - # for every key in attributes there are 6 possible cases: - # - (2) the key can either be present in the state or not - # - (3) the key can be a subgraph, a leaf, or a static attribute - for attribute in nodedef.attributes: - key = attribute.key - if key not in state: - # if key is not present create an empty types - if type(attribute) is StaticAttribute: - children.append((key, attribute.value)) - elif type(attribute) is SubGraphAttribute: - # if the key is a subgraph we create an empty node - subgraphdef = attribute.value - assert not isinstance(subgraphdef, VariableDef) - if isinstance(subgraphdef, NodeRef): - # subgraph exists, take it from the cache - children.append((key, index_ref[subgraphdef.index])) - else: - # create a node from an empty state, reasoning: - # * its a node with no state - # * its a node with state but only through references of already - # created nodes - substate = {} - subnode = _graph_unflatten( - subgraphdef, substate, index_ref, index_ref_cache - ) - children.append((key, subnode)) - elif type(attribute) is LeafAttribute: - variabledef = attribute.value - if variabledef.index in index_ref: - # variable exists, take it from the cache - children.append((key, index_ref[variabledef.index])) - else: - # key for a variable is missing, raise an error + + assert type(nodedef) is NodeDef + for key, value in nodedef.attributes: + if type(value) is Static: + children.append((key, value.value)) + elif type(value) is NodeRef: + children.append((key, index_ref[value.index])) + elif type(value) is NodeDef: + # if the key is a subgraph we create an empty node + subgraphdef = value + value_node_impl = get_node_impl_for_type(subgraphdef.type) + assert value_node_impl is not None + subnode = _graph_unflatten( + subgraphdef, value_node_impl, leaves, index_ref, outer_index_outer_ref + ) + children.append((key, subnode)) + elif type(value) is VariableDef: + variabledef = value + if not leaves: + raise ValueError('Not enough leaves to unflatten the graph') + # its a unseen variable, create a new one + value = leaves.popleft() + # when idxmap is present, check if the Varable exists there + # and update existing variables if it does + if ( + outer_index_outer_ref is not None + and variabledef.outer_index in outer_index_outer_ref + ): + # if variable exists, update it + variable = outer_index_outer_ref[variabledef.outer_index] + if not isinstance(variable, Variable): raise ValueError( - f'Expected key {key!r} in state while building node of type ' - f'{nodedef.type.__name__}.' + f'Expected a Variable type for {key!r}, but got {type(variable)}.' ) - else: - raise RuntimeError(f'Unknown static field: {key!r}') - else: - state_keys.remove(key) - value = state[key] - # if key in nodedef.static_fields: - if type(attribute) is StaticAttribute: - raise ValueError( - f'Got state for static field {key!r}, this is not supported.' - ) - elif type(attribute) is SubGraphAttribute: - if is_state_leaf(value): + elif isinstance(value, Variable): raise ValueError( - f'Expected value of type {attribute.value} for ' - f'{key!r}, but got {value!r}' + f'Cannot unflatten flat_state containing Variables when using `outer_index_outer_ref`. ' + f'Got {value!r} for {key!r}.' ) - assert isinstance(value, dict) - subgraphdef = attribute.value - - if isinstance(subgraphdef, NodeRef): - children.append((key, index_ref[subgraphdef.index])) + elif isinstance(value, VariableState): + variable.update_from_state(value) else: - subnode = _graph_unflatten( - subgraphdef, value, index_ref, index_ref_cache - ) - children.append((key, subnode)) - - elif type(attribute) is LeafAttribute: - variabledef = attribute.value - - if variabledef.index in index_ref: - # add an existing variable - assert isinstance(variabledef, NodeRef) - children.append((key, index_ref[variabledef.index])) + variable.raw_value = value + else: # variabledef.index not in index_ref_cache + # variable reference does not exist outside, create a new one + if isinstance(value, Variable): + variable = value + elif isinstance(value, VariableState): + variable = value.to_variable() else: - # its a unseen variable, create a new one - assert isinstance(variabledef, VariableDef) - # when idxmap is present, check if the Varable exists there - # and update existing variables if it does - if ( - index_ref_cache is not None - and variabledef.index in index_ref_cache - ): - # if variable exists, update it - variable = index_ref_cache[variabledef.index] - if not isinstance(variable, Variable): - raise ValueError( - f'Expected a Variable type for {key!r}, but got {type(variable)}.' - ) - if isinstance(value, VariableState): - variable.update_from_state(value) - else: - variable.raw_value = value - else: # if it doesn't, create a new variable - if isinstance(value, VariableState): - variable = value.to_variable() - else: - variable = variabledef.type.from_metadata( - value, variabledef.metadata - ) - children.append((key, variable)) - index_ref[variabledef.index] = variable - else: - raise RuntimeError(f'Unknown key: {key!r}, this is a bug.') - - # NOTE: we could allw adding new StateLeafs here - if state_keys: - raise ValueError(f'Unknown keys: {state_keys}') + variable = variabledef.type.from_metadata( + value, variabledef.metadata + ) + children.append((key, variable)) + index_ref[variabledef.index] = variable + else: + raise RuntimeError(f'Unknown static field: {key!r}') return children if isinstance(node_impl, GraphNodeImpl): # we create an empty node first and add it to the index # this avoids infinite recursion when there is a reference cycle - if index_ref_cache is not None and nodedef.index in index_ref_cache: - node = index_ref_cache[nodedef.index] + if ( + outer_index_outer_ref is not None + and nodedef.outer_index in outer_index_outer_ref + ): + node = outer_index_outer_ref[nodedef.outer_index] if type(node) != nodedef.type: raise ValueError( f'Expected a node of type {nodedef.type} for index ' @@ -765,26 +1074,176 @@ def _graph_update_dynamic(node: tp.Any, state: tp.Mapping[KeyT, tp.Any]): # updated from raw value current_value.raw_value = value + # -------------------------------------------------------- # UpdateContext # -------------------------------------------------------- + +class DynamicCache(tp.NamedTuple): + fingerprint: list[tp.Hashable] + graphdef: GraphDef[tp.Any] + final_graphdef: GraphDef[tp.Any] + paths: tuple[PathParts, ...] + variables: list[Variable[tp.Any]] + new_index_ref: dict[Index, tp.Any] + + @staticmethod + def create( + fingerprint: list[tp.Hashable], + graphdef: GraphDef[tp.Any], + paths: tuple[PathParts, ...], + variables: list[Variable[tp.Any]], + new_ref_index: RefMap, + ): + new_index_ref = {index: obj for obj, index in new_ref_index.items()} + if type(graphdef) is NodeDef: + final_graphdef = graphdef.with_same_outer_index() + else: + final_graphdef = graphdef + return DynamicCache( + fingerprint=fingerprint, + graphdef=graphdef, + final_graphdef=final_graphdef, + paths=paths, + variables=variables, + new_index_ref=new_index_ref, + ) + +class StaticCache(tp.NamedTuple): + graphdef: GraphDef[tp.Any] + final_graphdef: GraphDef[tp.Any] + paths: tuple[PathParts, ...] + variables: list[Variable[tp.Any]] + new_ref_index: RefMap + new_index_ref: dict[Index, tp.Any] + + @staticmethod + def create( + graphdef: GraphDef[tp.Any], + paths: tuple[PathParts, ...], + variables: list[Variable[tp.Any]], + new_ref_index: RefMap, + ): + new_index_ref = {index: obj for obj, index in new_ref_index.items()} + if type(graphdef) is NodeDef: + final_graphdef = graphdef.with_same_outer_index() + else: + final_graphdef = graphdef + return StaticCache( + graphdef=graphdef, + final_graphdef=final_graphdef, + paths=paths, + variables=variables, + new_ref_index=new_ref_index, + new_index_ref=new_index_ref, + ) + @dataclasses.dataclass class GraphContext(threading.local): - update_context_stacks: dict[str, list[UpdateContext]] = dataclasses.field( - default_factory=dict + update_context_stacks: dict[tp.Hashable, list[UpdateContext]] = ( + dataclasses.field(default_factory=dict) ) ref_index_stack: list[SplitContext] = dataclasses.field(default_factory=list) index_ref_stack: list[MergeContext] = dataclasses.field(default_factory=list) + dynamic_cache_context: WeakKeyDictionary[ + tp.Hashable, WeakKeyDictionary[tp.Any, DynamicCache] + ] = dataclasses.field(default_factory=WeakKeyDictionary) + tmp_dynamic_cache: WeakKeyDictionary[tp.Any, DynamicCache] | None = None + tmp_static_cache: WeakKeyDictionary[tp.Any, StaticCache] | None = None + caching: bool = False GRAPH_CONTEXT = GraphContext() +@contextlib.contextmanager +def dynamic_cache(ctxtag: tp.Hashable): + if GRAPH_CONTEXT.caching: + yield + return + + GRAPH_CONTEXT.caching = True + if ctxtag not in GRAPH_CONTEXT.dynamic_cache_context: + GRAPH_CONTEXT.dynamic_cache_context[ctxtag] = WeakKeyDictionary() + + current_dynamic_cache = GRAPH_CONTEXT.dynamic_cache_context[ctxtag] + GRAPH_CONTEXT.tmp_dynamic_cache = current_dynamic_cache + + try: + yield + finally: + if GRAPH_CONTEXT.tmp_dynamic_cache is not None: + raise ValueError( + 'GRAPH_CONTEXT.tmp_dynamic_cache should be None, no context consumed it.' + ) + GRAPH_CONTEXT.caching = False + +@contextlib.contextmanager +def static_cache(static_cache: WeakKeyDictionary[tp.Any, StaticCache]): + if GRAPH_CONTEXT.caching: + yield + return + + GRAPH_CONTEXT.tmp_static_cache = static_cache + + try: + yield + finally: + if GRAPH_CONTEXT.tmp_static_cache is not None: + raise ValueError( + 'GRAPH_CONTEXT.tmp_static_cache should be None, no context consumed it.' + ) + + +def _cache_args(f: tp.Callable[..., tp.Any], *cached_args): + cache: WeakKeyDictionary[tp.Any, StaticCache] = WeakKeyDictionary() + original_ref_index = RefMap() + index_ref: dict[Index, tp.Any] = {} + cached_ref_index = RefMap() + + def create_static_cache(x): + if is_graph_node(x): + graphdef, flat_state = flatten( + x, with_paths=True, return_variables=True, ref_index=original_ref_index + ) + paths = flat_state.get_keys() + variables = flat_state.get_values() + # clone but keep the same variable references + node_cache = unflatten(graphdef, flat_state, index_ref=index_ref) + cached_new_ref_index = RefMap() + _fp = fingerprint( + node_cache, + ref_index=cached_ref_index, + new_ref_index=cached_new_ref_index, + ) + cached_ref_index.update(cached_new_ref_index) + cache[node_cache] = StaticCache.create( + graphdef, paths, variables, cached_new_ref_index + ) + return node_cache + return x + + cached_args = jax.tree.map(create_static_cache, cached_args) + + @functools.wraps(f) + def cache_args_wrapper(*args, **kwargs): + with static_cache(cache): + return f(*cached_args, *args, **kwargs) + + return cache_args_wrapper + + +if tp.TYPE_CHECKING: + cache_args = functools.partial +else: + cache_args = _cache_args + @dataclasses.dataclass class SplitContext: - ctxtag: str | None - ref_index: RefMap[tp.Any, Index] + ctxtag: tp.Hashable | None + ref_index: RefMap + is_inner: bool | None @tp.overload def split(self, graph_node: A, /) -> tuple[GraphDef[A], GraphState]: ... @@ -807,31 +1266,185 @@ def split( ctx = ( current_update_context(self.ctxtag) if self.ctxtag is not None else None ) - graphdef, state = flatten(node, self.ref_index) - states = _split_state(state, filters) - if ctx is not None: - if ctx.index_ref is not None and isinstance(graphdef, NodeDef): - index_to_index = compose_mapping(ctx.index_ref, self.ref_index) - graphdef = dataclasses.replace( - graphdef, index_mapping=HashableMapping(index_to_index, copy=False) - ) + inner_ref_outer_index = ctx and ctx.inner_ref_outer_index + graphdef, flat_state = flatten( + node, ref_index=self.ref_index, ref_outer_index=inner_ref_outer_index + ) + flat_states = _split_state(flat_state, filters) + states = tuple( + State.from_flat_path(flat_state) for flat_state in flat_states + ) return graphdef, *states + @tp.overload + def flatten( + self, + graph_node: A, + /, + *, + with_paths: tp.Literal[False], + ) -> tuple[GraphDef[A], list[tp.Any]]: ... + @tp.overload + def flatten( + self, + graph_node: A, + /, + ) -> tuple[GraphDef[A], FlatState[VariableState[tp.Any]]]: ... + @tp.overload + def flatten( + self, + graph_node: A, + first: filterlib.Filter, + /, + ) -> tuple[GraphDef[A], FlatState[VariableState[tp.Any]]]: ... + @tp.overload + def flatten( + self, + graph_node: A, + first: filterlib.Filter, + second: filterlib.Filter, + /, + *filters: filterlib.Filter, + ) -> tuple[ + GraphDef[A], + FlatState[VariableState[tp.Any]], + tpe.Unpack[tuple[FlatState[VariableState[tp.Any]], ...]], + ]: ... + def flatten( + self, + node: A, + *filters: filterlib.Filter, + with_paths: bool = True, + ) -> tuple[ + GraphDef[A], + FlatState[VariableState[tp.Any]] | list[tp.Any], + tpe.Unpack[tuple[FlatState[VariableState[tp.Any]], ...]], + ]: + if not with_paths and filters: + raise ValueError('Cannot use filters with with_paths=False') + + ctx = ( + current_update_context(self.ctxtag) if self.ctxtag is not None else None + ) + dynamic_cache = ( + ctx.dynamic_cache if ctx is not None and self.is_inner is False else None + ) + static_cache = ( + ctx.static_cache if ctx is not None and self.is_inner is False else None + ) + ref_outer_index = ctx and ctx.inner_ref_outer_index + + if node in self.ref_index: + # node is already in the ref_index, call flatten which will return a NodeRef + return flatten( + node, ref_index=self.ref_index, ref_outer_index=ref_outer_index + ) + elif static_cache is not None and node in static_cache: + node_cache = static_cache[node] + graphdef = node_cache.graphdef + # add the new references to the ref_index + self.ref_index.update(node_cache.new_ref_index) + + if with_paths: + paths = node_cache.paths + leaves = [variable.to_state() for variable in node_cache.variables] + else: + paths = None + leaves = [variable.raw_value for variable in node_cache.variables] + + elif dynamic_cache is not None and node in dynamic_cache: + node_cache = dynamic_cache[node] + cache_fp = node_cache.fingerprint + new_ref_index = RefMap() + fp_matches = check_fingerprint( + node, cache_fp, ref_index=self.ref_index, new_ref_index=new_ref_index + ) + if fp_matches: + graphdef = node_cache.graphdef + self.ref_index.update(new_ref_index) + + if with_paths: + paths = node_cache.paths + leaves = [variable.to_state() for variable in node_cache.variables] + else: + paths = None + leaves = [variable.raw_value for variable in node_cache.variables] + else: + del cache_fp + del node_cache + new_ref_index = RefMap() + node_fp = fingerprint( + node, ref_index=self.ref_index, new_ref_index=new_ref_index + ) + graphdef, flat_states = flatten( + node, + ref_index=self.ref_index, + ref_outer_index=ref_outer_index, + with_paths=True, + return_variables=True, + ) + paths = flat_states.get_keys() + variables = flat_states.get_values() + assert paths is not None + if with_paths: + leaves = [variable.to_state() for variable in variables] + else: + leaves = [variable.raw_value for variable in variables] + dynamic_cache[node] = DynamicCache.create( + node_fp, graphdef, paths, variables, new_ref_index + ) + elif dynamic_cache is not None: # node not in cache_context + new_ref_index = RefMap() + node_fp = fingerprint( + node, ref_index=self.ref_index, new_ref_index=new_ref_index + ) + graphdef, flat_state = flatten( + node, + ref_index=self.ref_index, + ref_outer_index=ref_outer_index, + with_paths=True, + return_variables=True, + ) + paths = flat_state.get_keys() + variables = flat_state.get_values() + if with_paths: + leaves = [variable.to_state() for variable in variables] + else: + leaves = [variable.raw_value for variable in variables] + dynamic_cache[node] = DynamicCache.create( + node_fp, graphdef, paths, variables, new_ref_index + ) + else: + return flatten( + node, + ref_index=self.ref_index, + with_paths=with_paths, + ref_outer_index=ref_outer_index, + ) + + if with_paths: + assert paths is not None + flat_state = FlatState.from_sorted_keys_values(paths, leaves) + flat_states = _split_state(flat_state, filters) + return graphdef, *flat_states + else: + return graphdef, leaves + @contextlib.contextmanager -def split_context(ctxtag: str | None = None): - index_ref: RefMap[tp.Any, Index] = RefMap() - flatten_ctx = SplitContext(ctxtag, index_ref) - GRAPH_CONTEXT.ref_index_stack.append(flatten_ctx) +def split_context(ctxtag: tp.Hashable | None = None): + ctx = current_update_context(ctxtag) if ctxtag is not None else None + is_inner = ctx.outer_ref_outer_index is not None if ctx is not None else None + GRAPH_CONTEXT.ref_index_stack.append(SplitContext(ctxtag, RefMap(), is_inner)) try: - yield flatten_ctx + yield GRAPH_CONTEXT.ref_index_stack[-1] finally: - GRAPH_CONTEXT.ref_index_stack.pop() + flatten_ctx = GRAPH_CONTEXT.ref_index_stack.pop() if ctxtag is not None: ctx = current_update_context(ctxtag) - ctx.flatten_end(index_ref) + ctx.flatten_end(flatten_ctx.ref_index) del flatten_ctx.ref_index del flatten_ctx.ctxtag @@ -840,51 +1453,166 @@ def split_context(ctxtag: str | None = None): class MergeContext: ctxtag: str | None index_ref: dict[Index, tp.Any] + is_inner: bool | None def merge( - self, graphdef: GraphDef[A], state: GraphState, /, *states: GraphState + self, + graphdef: GraphDef[A], + state: GraphState, + /, + *states: GraphState, ) -> A: ctx = ( current_update_context(self.ctxtag) if self.ctxtag is not None else None ) - if ( - ctx is not None - and isinstance(graphdef, NodeDef) - and graphdef.index_mapping is not None - ): - # outer merge (4), create index_ref_cache - assert ctx.ref_index is not None - index_ref_cache = compose_mapping_reversed( - ctx.ref_index, graphdef.index_mapping - ) - else: - # inner merge (2) - index_ref_cache = None state = State.merge(state, *states) node = unflatten( graphdef, state, index_ref=self.index_ref, - index_ref_cache=index_ref_cache, + outer_index_outer_ref=ctx and ctx.outer_index_outer_ref, ) return node + def unflatten( + self, + graphdef: GraphDef[A], + flat_state: GraphFlatState | list[tp.Any], + /, + *flat_states: GraphFlatState, + ) -> A: + ctx = ( + current_update_context(self.ctxtag) if self.ctxtag is not None else None + ) + dynamic_cache = ( + ctx.dynamic_cache if ctx is not None and self.is_inner is False else None + ) + static_cache = ( + ctx.static_cache if ctx is not None and self.is_inner is False else None + ) -@contextlib.contextmanager -def merge_context(ctxtag: str | None = None): - index_ref: dict[Index, tp.Any] = {} + if type(flat_state) is list: + if flat_states: + raise ValueError( + 'Cannot use multiple flat_states when flat_state is a list, ' + f'got flat_state: {flat_state!r}, flat_states: {flat_states!r}' + ) + state = flat_state + else: + state = FlatState.merge(flat_state, *flat_states) + + if type(graphdef) is NodeRef: + node = unflatten( + graphdef, + state, + index_ref=self.index_ref, + ) + + elif dynamic_cache is not None or static_cache is not None: + assert isinstance(graphdef, NodeDef) + assert ctx is not None + if (outer_index := graphdef.outer_index) is not None: + outer_index_outer_ref = ctx.outer_index_outer_ref + assert outer_index_outer_ref is not None + node = outer_index_outer_ref[outer_index] + + if static_cache and node in static_cache: + cache = static_cache[node] + if cache.final_graphdef != graphdef: + raise ValueError( + 'The graph structure of a node added to cache_args was mutated inside the transformation, ' + f'this is not allowed.\nNode: {node}\nOuput graphdef: {graphdef}\nExpected graphdef: {cache.final_graphdef}' + ) + if type(state) is list: + leaves = state + elif type(state) is FlatState: + leaves = state.get_values() + else: + raise ValueError(f'Unsupported state type: {type(state)}') + + if len(leaves) != len(cache.variables): + raise ValueError( + f'Incorrect number of leaves: expected {len(cache.variables)} ' + f'leaves in the state, got {len(leaves)}' + ) + for variable, leaf in zip(cache.variables, leaves): + if type(leaf) is VariableState: + variable.update_from_state(leaf) + else: + variable.raw_value = leaf + self.index_ref.update(cache.new_index_ref) + elif dynamic_cache and node in dynamic_cache: + # node is in cache_context, retrieve its cache + cache = dynamic_cache[node] + # check if the graphdef is the same + if cache.final_graphdef == graphdef: + if type(state) is list: + leaves = state + elif type(state) is FlatState: # type: ignore + leaves = state.get_values() + else: + raise ValueError(f'Unsupported state type: {type(state)}') + + # graphdefs match, update variables from state + if len(leaves) != len(cache.variables): + raise ValueError( + f'Incorrect number of leaves: expected {len(cache.variables)} ' + f'leaves in the state, got {len(leaves)}' + ) + for variable, leaf in zip(cache.variables, leaves): + if type(leaf) is VariableState: + variable.update_from_state(leaf) + else: + variable.raw_value = leaf + self.index_ref.update(cache.new_index_ref) + else: # cache.graphdef != graphdef_fp + # graph changed, re-create the node + node = unflatten( + graphdef, + state, + index_ref=self.index_ref, + outer_index_outer_ref=outer_index_outer_ref, + ) + else: + # all nodes in index_ref_cache must be in cache_context + raise RuntimeError(f'Node not found in cache_context, node: {node}') + else: # graphdef.outer_index is None + # its a new node, create it + node = unflatten( + graphdef, + state, + index_ref=self.index_ref, + ) + else: + node = unflatten( + graphdef, + state, + index_ref=self.index_ref, + outer_index_outer_ref=ctx and ctx.outer_index_outer_ref, + ) + return node - unflatten_ctx = MergeContext(ctxtag, index_ref) - GRAPH_CONTEXT.index_ref_stack.append(unflatten_ctx) +@tp.overload +@contextlib.contextmanager +def merge_context(): ... +@tp.overload +@contextlib.contextmanager +def merge_context(inner: bool | None, ctxtag: str | None): ... +@contextlib.contextmanager +def merge_context(inner: bool | None = None, ctxtag: str | None = None): + GRAPH_CONTEXT.index_ref_stack.append(MergeContext(ctxtag, {}, inner)) try: - yield unflatten_ctx + yield GRAPH_CONTEXT.index_ref_stack[-1] finally: - GRAPH_CONTEXT.index_ref_stack.pop() + unflatten_ctx = GRAPH_CONTEXT.index_ref_stack.pop() + index_ref = unflatten_ctx.index_ref if ctxtag is not None: + if inner is None: + raise ValueError('inner_merge must be specified when using ctxtag') ctx = current_update_context(ctxtag) - ctx.unflatten_end(index_ref) + ctx.unflatten_end(index_ref, inner) del unflatten_ctx.index_ref del unflatten_ctx.ctxtag @@ -893,9 +1621,14 @@ def merge_context(ctxtag: str | None = None): class UpdateContext: """A context manager for handling complex state updates.""" - tag: str - ref_index: RefMap[tp.Any, Index] | None - index_ref: dict[Index, tp.Any] | None + tag: tp.Hashable + outer_ref_outer_index: RefMap | None + outer_index_inner_ref: dict[Index, tp.Any] | None + # reverse caches + outer_index_outer_ref: dict[Index, tp.Any] | None + inner_ref_outer_index: RefMap | None + dynamic_cache: WeakKeyDictionary[tp.Any, DynamicCache] | None + static_cache: WeakKeyDictionary[tp.Any, StaticCache] | None # define hash and eq to make this an opaque object def __hash__(self): @@ -904,16 +1637,25 @@ def __hash__(self): def __eq__(self, other): return isinstance(other, UpdateContext) - def flatten_end(self, ref_index: RefMap[tp.Any, Index]): - if self.ref_index is None: + def flatten_end(self, ref_index: RefMap): + if self.outer_ref_outer_index is None: # outer split (1), store the references - self.ref_index = ref_index + self.outer_ref_outer_index = ref_index + self.outer_index_outer_ref = { + index: obj for obj, index in self.outer_ref_outer_index.items() + } else: # inner split (3), clear index_ref - self.index_ref = None + self.outer_index_inner_ref = None + self.inner_ref_outer_index = None - def unflatten_end(self, index_ref: dict[Index, tp.Any]): - self.index_ref = index_ref + def unflatten_end(self, index_ref: dict[Index, tp.Any], inner_merge: bool): + if inner_merge: + # inner merge (2) + self.outer_index_inner_ref = index_ref + self.inner_ref_outer_index = RefMap( + {obj: index for index, obj in index_ref.items()} + ) @tp.overload def split(self, graph_node: A, /) -> tuple[GraphDef[A], GraphState]: ... @@ -996,15 +1738,14 @@ def split( :class:`GraphDef` and one or more :class:`State`'s equal to the number of filters passed. If no filters are passed, a single :class:`State` is returned. """ - ref_index: RefMap[tp.Any, Index] = RefMap() - graphdef, state = flatten(node, ref_index) - states = _split_state(state, filters) - - if self.index_ref is not None and isinstance(graphdef, NodeDef): - index_to_index = compose_mapping(self.index_ref, ref_index) - graphdef = dataclasses.replace( - graphdef, index_mapping=HashableMapping(index_to_index, copy=False) - ) + ref_index: RefMap = RefMap() + graphdef, flat_state = flatten( + node, ref_index=ref_index, ref_outer_index=self.inner_ref_outer_index + ) + states = tuple( + State.from_flat_path(flat_state) + for flat_state in _split_state(flat_state, filters) + ) self.flatten_end(ref_index) @@ -1021,15 +1762,13 @@ def merge( raise ValueError( f'Expected a NodeDef instance, but got {type(graphdef)}.' ) - if self.ref_index is None: + if self.outer_ref_outer_index is None: raise ValueError('Cannot merge without ref_index.') - if graphdef.index_mapping is not None: + if self.outer_ref_outer_index is not None: # outer merge (4), create index_ref_cache - assert self.ref_index is not None - index_ref_cache = compose_mapping_reversed( - self.ref_index, graphdef.index_mapping - ) + index_ref_cache = self.outer_index_outer_ref + assert index_ref_cache is not None else: # inner merge (2) index_ref_cache = None @@ -1037,10 +1776,13 @@ def merge( state = State.merge(state, *states) index_ref: dict[Index, tp.Any] = {} node = unflatten( - graphdef, state, index_ref=index_ref, index_ref_cache=index_ref_cache + graphdef, + state, + index_ref=index_ref, + outer_index_outer_ref=index_ref_cache, ) - self.unflatten_end(index_ref) + self.unflatten_end(index_ref, True) return node @@ -1050,10 +1792,30 @@ def merge( @dataclasses.dataclass class UpdateContextManager: - tag: str + tag: tp.Hashable def __enter__(self): - ctx = UpdateContext(self.tag, None, None) + if GRAPH_CONTEXT.tmp_dynamic_cache is not None: + # take current dynamic cache + dynamic_cache = GRAPH_CONTEXT.tmp_dynamic_cache + GRAPH_CONTEXT.tmp_dynamic_cache = None + else: + dynamic_cache = None + if GRAPH_CONTEXT.tmp_static_cache is not None: + # take current static cache + static_cache = GRAPH_CONTEXT.tmp_static_cache + GRAPH_CONTEXT.tmp_static_cache = None + else: + static_cache = None + ctx = UpdateContext( + tag=self.tag, + outer_ref_outer_index=None, + outer_index_inner_ref=None, + outer_index_outer_ref=None, + inner_ref_outer_index=None, + dynamic_cache=dynamic_cache, + static_cache=static_cache, + ) if self.tag not in GRAPH_CONTEXT.update_context_stacks: GRAPH_CONTEXT.update_context_stacks[self.tag] = [ctx] else: @@ -1069,8 +1831,10 @@ def __exit__(self, *args): ctx = stack.pop() # clear references - del ctx.ref_index - del ctx.index_ref + del ctx.outer_ref_outer_index + del ctx.outer_index_inner_ref + del ctx.outer_index_outer_ref + del ctx.inner_ref_outer_index if not stack: del GRAPH_CONTEXT.update_context_stacks[self.tag] @@ -1084,7 +1848,7 @@ def update_context_manager_wrapper(*args, **kwargs): return update_context_manager_wrapper # type: ignore -def update_context(tag: str): +def update_context(tag: tp.Hashable): """Creates an :class:`UpdateContext` context manager which can be used to handle more complex state updates beyond what ``nnx.update`` can handle, including updates to static properties and graph structure. @@ -1179,7 +1943,7 @@ def update_context(tag: str): return UpdateContextManager(tag) -def current_update_context(tag: str) -> UpdateContext: +def current_update_context(tag: tp.Hashable) -> UpdateContext: """Returns the current active :class:`UpdateContext` for the given tag.""" if tag not in GRAPH_CONTEXT.update_context_stacks: raise ValueError(f'No update context found for tag {tag!r}.') @@ -1191,13 +1955,13 @@ def current_update_context(tag: str) -> UpdateContext: # -------------------------------------------------------- def _split_state( - state: GraphState, + state: FlatState[tp.Any], filters: tuple[filterlib.Filter, ...], -) -> tuple[GraphState, tpe.Unpack[tuple[GraphState, ...]]]: +) -> tuple[FlatState[tp.Any], tpe.Unpack[tuple[FlatState[tp.Any], ...]]]: if not filters: return (state,) states = state.split(*filters) - if isinstance(states, State): + if not isinstance(states, tuple): return (states,) assert len(states) > 0 return states # type: ignore[return-value] @@ -1288,9 +2052,11 @@ def split( ``GraphDef`` and one or more ``States`` equal to the number of filters passed. If no filters are passed, a single ``State`` is returned. """ - graphdef, state = flatten(node) - states = _split_state(state, filters) - return graphdef, *states + graphdef, flat_state = flatten(node) + flat_states = _split_state(flat_state, filters) + states = tuple(State.from_flat_path(flat_state) for flat_state in flat_states) + return graphdef, *states # type: ignore[return-value] + def merge( graphdef: GraphDef[A], @@ -1482,6 +2248,7 @@ def state( One or more :class:`State` mappings. """ _, state = flatten(node) + state = state.to_nested_state() states: GraphState | tuple[GraphState, ...] if len(filters) == 0: @@ -1755,16 +2522,6 @@ def _iter_graph( yield path_parts, node -def compose_mapping( - map_ab: tp.Mapping[A, B], map_bc: tp.Mapping[B, C], / -) -> dict[A, C]: - return {a: map_bc[b] for a, b in map_ab.items() if b in map_bc} - - -def compose_mapping_reversed( - map_ab: tp.Mapping[A, B], map_bc: tp.Mapping[B, C], / -) -> dict[C, A]: - return {map_bc[b]: a for a, b in map_ab.items() if b in map_bc} @dataclasses.dataclass(frozen=True) @@ -1783,21 +2540,15 @@ class Static(tp.Generic[A]): # --------------------------------------------------------- class GenericPytree: ... +from jax._src.tree_util import _registry as JAX_PYTREE_REGISTRY def is_pytree_node(x: tp.Any) -> bool: - t = type(x) - if t in PYTREE_REGISTRY: + if type(x) in JAX_PYTREE_REGISTRY: return True - elif t in GRAPH_REGISTRY: - return False - # known non-pytree types - elif isinstance(x, Variable): - return False - # known pytree types - elif type(x) is VariableState or type(x) is State: + elif isinstance(x, tuple): return True else: - return not jax.tree_util.all_leaves((x,)) + return False def _key_path_to_key(key: tp.Any) -> Key: @@ -1816,20 +2567,28 @@ def _key_path_to_key(key: tp.Any) -> Key: else: return str(key) +class IndexesPytreeDef(tp.NamedTuple): + key_index: HashableMapping[Key, int] + treedef: jax.tree_util.PyTreeDef def _flatten_pytree(pytree: tp.Any): leaves, treedef = jax.tree_util.tree_flatten_with_path( pytree, is_leaf=lambda x: x is not pytree ) - nodes = tuple((_key_path_to_key(path[0]), value) for path, value in leaves) - - return nodes, treedef + nodes = [(_key_path_to_key(path[0]), value) for path, value in leaves] + key_index = HashableMapping( + {key: i for i, (key, _) in enumerate(nodes)}, copy=False + ) + nodes.sort() # sort by key + return nodes, IndexesPytreeDef(key_index, treedef) def _unflatten_pytree( - nodes: tuple[tuple[Key, tp.Any], ...], treedef: jax.tree_util.PyTreeDef + nodes: tuple[tuple[Key, tp.Any], ...], metadata: IndexesPytreeDef ): - pytree = treedef.unflatten(value for _, value in nodes) + # sort to original order + sorted_nodes = sorted(nodes, key=lambda x: metadata.key_index[x[0]]) + pytree = metadata.treedef.unflatten(value for _, value in sorted_nodes) return pytree diff --git a/flax/nnx/helpers.py b/flax/nnx/helpers.py index 96622f0e40..077817c4a1 100644 --- a/flax/nnx/helpers.py +++ b/flax/nnx/helpers.py @@ -62,6 +62,10 @@ def __iter__(self) -> tp.Iterator[str]: def __len__(self) -> int: return len(vars(self)) + def __hash__(self) -> int: + return id(self) + + class Sequential(Module): def __init__(self, *fns: tp.Callable[..., tp.Any]): self.layers = list(fns) diff --git a/flax/nnx/module.py b/flax/nnx/module.py index b07efa7711..795bb9a088 100644 --- a/flax/nnx/module.py +++ b/flax/nnx/module.py @@ -403,6 +403,23 @@ def __init_subclass__(cls, experimental_pytree: bool = False) -> None: flatten_func=partial(_module_flatten, with_keys=False), ) + def __treescope_repr__(self, path, subtree_renderer): + import treescope # type: ignore[import-not-found,import-untyped] + children = {} + for name, value in vars(self).items(): + if name.startswith('_'): + continue + children[name] = value + return treescope.repr_lib.render_object_constructor( + object_type=type(self), + attributes=children, + path=path, + subtree_renderer=subtree_renderer, + color=treescope.formatting_util.color_from_string( + type(self).__qualname__ + ) + ) + # ------------------------- # Pytree Definition # ------------------------- diff --git a/flax/nnx/nn/linear.py b/flax/nnx/nn/linear.py index 230f1d356e..364b5dac1e 100644 --- a/flax/nnx/nn/linear.py +++ b/flax/nnx/nn/linear.py @@ -1063,7 +1063,7 @@ class Embed(Module): >>> layer = nnx.Embed(num_embeddings=5, features=3, rngs=nnx.Rngs(0)) >>> nnx.state(layer) State({ - 'embedding': VariableState( # 15 (60 B) + 'embedding': VariableState( type=Param, value=Array([[-0.90411377, -0.3648777 , -1.1083648 ], [ 0.01070483, 0.27923733, 1.7487359 ], diff --git a/flax/nnx/nn/normalization.py b/flax/nnx/nn/normalization.py index 928d9cf251..b5cbaf99b6 100644 --- a/flax/nnx/nn/normalization.py +++ b/flax/nnx/nn/normalization.py @@ -395,11 +395,11 @@ class LayerNorm(Module): >>> nnx.state(layer) State({ - 'bias': VariableState( # 6 (24 B) + 'bias': VariableState( type=Param, value=Array([0., 0., 0., 0., 0., 0.], dtype=float32) ), - 'scale': VariableState( # 6 (24 B) + 'scale': VariableState( type=Param, value=Array([1., 1., 1., 1., 1., 1.], dtype=float32) ) @@ -531,7 +531,7 @@ class RMSNorm(Module): >>> nnx.state(layer) State({ - 'scale': VariableState( # 6 (24 B) + 'scale': VariableState( type=Param, value=Array([1., 1., 1., 1., 1., 1.], dtype=float32) ) @@ -655,11 +655,11 @@ class GroupNorm(Module): >>> layer = nnx.GroupNorm(num_features=6, num_groups=3, rngs=nnx.Rngs(0)) >>> nnx.state(layer) State({ - 'bias': VariableState( # 6 (24 B) + 'bias': VariableState( type=Param, value=Array([0., 0., 0., 0., 0., 0.], dtype=float32) ), - 'scale': VariableState( # 6 (24 B) + 'scale': VariableState( type=Param, value=Array([1., 1., 1., 1., 1., 1.], dtype=float32) ) diff --git a/flax/nnx/nn/recurrent.py b/flax/nnx/nn/recurrent.py index 6ce039c5b9..ea18805d0f 100644 --- a/flax/nnx/nn/recurrent.py +++ b/flax/nnx/nn/recurrent.py @@ -14,8 +14,10 @@ """RNN modules for Flax.""" -from typing import Any, TypeVar -from collections.abc import Mapping +from typing import ( + Any, + TypeVar +) from collections.abc import Callable from functools import partial from typing_extensions import Protocol @@ -25,13 +27,13 @@ import jax.numpy as jnp from flax import nnx -from flax.nnx import filterlib, rnglib +from flax.nnx import rnglib from flax.nnx.module import Module from flax.nnx.nn import initializers from flax.nnx.nn.linear import Linear from flax.nnx.nn.activations import sigmoid from flax.nnx.nn.activations import tanh -from flax.nnx.transforms.iteration import Carry, StateAxes +from flax.nnx.transforms.iteration import Carry from flax.typing import ( Dtype, Initializer, @@ -591,19 +593,15 @@ class RNN(Module): using :func:`flax.nnx.scan`. """ - state_axes: Mapping[str, int | type[Carry] | None] - def __init__( - self, - cell: RNNCellBase, - time_major: bool = False, - return_carry: bool = False, - reverse: bool = False, - keep_order: bool = False, - unroll: int = 1, - rngs: rnglib.Rngs | None = None, - state_axes: Mapping[str, int | type[Carry] | None] | None = None, - broadcast_rngs: filterlib.Filter = None, + self, + cell: RNNCellBase, + time_major: bool = False, + return_carry: bool = False, + reverse: bool = False, + keep_order: bool = False, + unroll: int = 1, + rngs: rnglib.Rngs | None = None, ): self.cell = cell self.time_major = time_major @@ -614,21 +612,19 @@ def __init__( if rngs is None: rngs = rnglib.Rngs(0) self.rngs = rngs - self.state_axes = state_axes or {...: Carry} # type: ignore - self.broadcast_rngs = broadcast_rngs def __call__( - self, - inputs: Array, - *, - initial_carry: Carry | None = None, - seq_lengths: Array | None = None, - return_carry: bool | None = None, - time_major: bool | None = None, - reverse: bool | None = None, - keep_order: bool | None = None, - rngs: rnglib.Rngs | None = None, - ): + self, + inputs: Array, + *, + initial_carry: Carry | None = None, + seq_lengths: Array | None = None, + return_carry: bool | None = None, + time_major: bool | None = None, + reverse: bool | None = None, + keep_order: bool | None = None, + rngs: rnglib.Rngs | None = None, + ): if return_carry is None: return_carry = self.return_carry if time_major is None: @@ -674,26 +670,20 @@ def __call__( ) slice_carry = seq_lengths is not None and return_carry - broadcast_rngs = nnx.All(nnx.RngState, self.broadcast_rngs) - state_axes = StateAxes({broadcast_rngs: None, **self.state_axes}) # type: ignore - - # we use split_rngs with splits=1 and squeeze=True to get unique rngs - # every time RNN is called - @nnx.split_rngs(splits=1, only=self.broadcast_rngs, squeeze=True) - @nnx.scan( - in_axes=(state_axes, Carry, time_axis), - out_axes=(Carry, (0, time_axis)) if slice_carry else (Carry, time_axis), - unroll=self.unroll, - ) - def scan_fn( - cell: RNNCellBase, carry: Carry, x: Array - ) -> tuple[Carry, Array] | tuple[Carry, tuple[Carry, Array]]: + + def scan_fn(cell: RNNCellBase, carry: Carry, x: Array) -> tuple[Carry, Array] | tuple[Carry, tuple[Carry, Array]]: carry, y = cell(carry, x) if slice_carry: return carry, (carry, y) return carry, y - - scan_output = scan_fn(self.cell, carry, inputs) + state_axes = nnx.StateAxes({...: Carry}) # type: ignore[arg-type] + scan = nnx.scan( + scan_fn, + in_axes=(state_axes, Carry, time_axis), + out_axes=(Carry, (0, time_axis)) if slice_carry else (Carry, time_axis), + unroll=self.unroll, + ) + scan_output = scan(self.cell, carry, inputs) # Next we select the final carry. If a segmentation mask was provided and # return_carry is True we slice the carry history and select the last valid diff --git a/flax/nnx/nn/stochastic.py b/flax/nnx/nn/stochastic.py index add545634a..737c6e3102 100644 --- a/flax/nnx/nn/stochastic.py +++ b/flax/nnx/nn/stochastic.py @@ -24,7 +24,7 @@ from flax.nnx.module import Module, first_from -@dataclasses.dataclass(repr=False) +@dataclasses.dataclass class Dropout(Module): """Create a dropout layer. @@ -125,3 +125,6 @@ def __call__( mask = random.bernoulli(rng, p=keep_prob, shape=broadcast_shape) mask = jnp.broadcast_to(mask, inputs.shape) return lax.select(mask, inputs / keep_prob, jnp.zeros_like(inputs)) + + def __hash__(self): + return id(self) diff --git a/flax/nnx/object.py b/flax/nnx/object.py index b1f7478eef..afa41cdb7b 100644 --- a/flax/nnx/object.py +++ b/flax/nnx/object.py @@ -20,67 +20,27 @@ from abc import ABCMeta from copy import deepcopy + import jax import numpy as np -import treescope # type: ignore[import-untyped] -from treescope import rendering_parts -from flax.nnx import visualization -from flax import errors from flax.nnx import ( - graph, reprlib, tracers, ) -from flax import nnx +from flax.nnx import graph from flax.nnx.variablelib import Variable, VariableState -from flax.typing import SizeBytes, value_stats +from flax import errors G = tp.TypeVar('G', bound='Object') -def _collect_stats( - node: tp.Any, node_stats: dict[int, dict[type[Variable], SizeBytes]] -): - if not graph.is_node(node) and not isinstance(node, Variable): - raise ValueError(f'Expected a graph node or Variable, got {type(node)!r}.') - - if id(node) in node_stats: - return - - stats: dict[type[Variable], SizeBytes] = {} - node_stats[id(node)] = stats - - if isinstance(node, Variable): - var_type = type(node) - if issubclass(var_type, nnx.RngState): - var_type = nnx.RngState - size_bytes = value_stats(node.value) - if size_bytes: - stats[var_type] = size_bytes - - else: - node_dict = graph.get_node_impl(node).node_dict(node) - for key, value in node_dict.items(): - if id(value) in node_stats: - continue - if graph.is_node(value) or isinstance(value, Variable): - _collect_stats(value, node_stats) - child_stats = node_stats[id(value)] - for var_type, size_bytes in child_stats.items(): - if var_type in stats: - stats[var_type] += size_bytes - else: - stats[var_type] = size_bytes - - @dataclasses.dataclass -class ObjectContext(threading.local): +class GraphUtilsContext(threading.local): seen_modules_repr: set[int] | None = None - node_stats: dict[int, dict[type[Variable], SizeBytes]] | None = None -OBJECT_CONTEXT = ObjectContext() +CONTEXT = GraphUtilsContext() class ObjectState(reprlib.Representable): @@ -103,14 +63,14 @@ def __nnx_repr__(self): yield reprlib.Attr('trace_state', self._trace_state) def __treescope_repr__(self, path, subtree_renderer): - return visualization.render_object_constructor( - object_type=type(self), - attributes={'trace_state': self._trace_state}, - path=path, - subtree_renderer=subtree_renderer, + import treescope # type: ignore[import-not-found,import-untyped] + return treescope.repr_lib.render_object_constructor( + object_type=type(self), + attributes={'trace_state': self._trace_state}, + path=path, + subtree_renderer=subtree_renderer, ) - class ObjectMeta(ABCMeta): if not tp.TYPE_CHECKING: @@ -130,14 +90,12 @@ def _graph_node_meta_call(cls: tp.Type[G], *args, **kwargs) -> G: @dataclasses.dataclass(frozen=True, repr=False) -class Array(reprlib.Representable): +class Array: shape: tp.Tuple[int, ...] dtype: tp.Any - def __nnx_repr__(self): - yield reprlib.Object(type='Array', same_line=True) - yield reprlib.Attr('shape', self.shape) - yield reprlib.Attr('dtype', self.dtype) + def __repr__(self): + return f'Array(shape={self.shape}, dtype={self.dtype.name})' class Object(reprlib.Representable, metaclass=ObjectMeta): @@ -179,41 +137,20 @@ def __deepcopy__(self: G, memo=None) -> G: return graph.merge(graphdef, state) def __nnx_repr__(self): - if OBJECT_CONTEXT.node_stats is None: - node_stats: dict[int, dict[type[Variable], SizeBytes]] = {} - _collect_stats(self, node_stats) - OBJECT_CONTEXT.node_stats = node_stats - stats = node_stats[id(self)] - clear_node_stats = True - else: - stats = OBJECT_CONTEXT.node_stats[id(self)] - clear_node_stats = False - - if OBJECT_CONTEXT.seen_modules_repr is None: - OBJECT_CONTEXT.seen_modules_repr = set() + if CONTEXT.seen_modules_repr is None: + CONTEXT.seen_modules_repr = set() clear_seen = True else: clear_seen = False - if id(self) in OBJECT_CONTEXT.seen_modules_repr: + if id(self) in CONTEXT.seen_modules_repr: yield reprlib.Object(type=type(self), empty_repr='...') return - try: - if stats: - stats_repr = ' # ' + ', '.join( - f'{var_type.__name__}: {size_bytes}' - for var_type, size_bytes in stats.items() - ) - if len(stats) > 1: - total_bytes = sum(stats.values(), SizeBytes(0, 0)) - stats_repr += f', Total: {total_bytes}' - else: - stats_repr = '' - - yield reprlib.Object(type=type(self), comment=stats_repr) - OBJECT_CONTEXT.seen_modules_repr.add(id(self)) + yield reprlib.Object(type=type(self)) + CONTEXT.seen_modules_repr.add(id(self)) + try: for name, value in vars(self).items(): if name.startswith('_'): continue @@ -231,64 +168,24 @@ def to_shape_dtype(value): return value value = jax.tree.map(to_shape_dtype, value) - yield reprlib.Attr(name, value) + yield reprlib.Attr(name, repr(value)) finally: if clear_seen: - OBJECT_CONTEXT.seen_modules_repr = None - if clear_node_stats: - OBJECT_CONTEXT.node_stats = None + CONTEXT.seen_modules_repr = None def __treescope_repr__(self, path, subtree_renderer): - from flax import nnx - - if OBJECT_CONTEXT.node_stats is None: - node_stats: dict[int, dict[type[Variable], SizeBytes]] = {} - _collect_stats(self, node_stats) - OBJECT_CONTEXT.node_stats = node_stats - stats = node_stats[id(self)] - clear_node_stats = True - else: - stats = OBJECT_CONTEXT.node_stats[id(self)] - clear_node_stats = False - - try: - if stats: - stats_repr = ' # ' + ', '.join( - f'{var_type.__name__}: {size_bytes}' - for var_type, size_bytes in stats.items() - ) - if len(stats) > 1: - total_bytes = sum(stats.values(), SizeBytes(0, 0)) - stats_repr += f', Total: {total_bytes}' - - first_line_annotation = rendering_parts.comment_color( - rendering_parts.text(f'{stats_repr}') - ) - else: - first_line_annotation = None - children = {} - for name, value in vars(self).items(): - if name.startswith('_'): - continue - children[name] = value - - if isinstance(self, nnx.Module): - color = treescope.formatting_util.color_from_string( - type(self).__qualname__ - ) - else: - color = None - return visualization.render_object_constructor( + import treescope # type: ignore[import-not-found,import-untyped] + children = {} + for name, value in vars(self).items(): + if name.startswith('_'): + continue + children[name] = value + return treescope.repr_lib.render_object_constructor( object_type=type(self), attributes=children, path=path, subtree_renderer=subtree_renderer, - first_line_annotation=first_line_annotation, - color=color, - ) - finally: - if clear_node_stats: - OBJECT_CONTEXT.node_stats = None + ) # Graph Definition def _graph_node_flatten(self): @@ -328,4 +225,4 @@ def _graph_node_clear(self): module_vars['_object__state'] = module_state def _graph_node_init(self, attributes: tp.Iterable[tuple[str, tp.Any]]): - vars(self).update(attributes) + vars(self).update(attributes) \ No newline at end of file diff --git a/flax/nnx/reprlib.py b/flax/nnx/reprlib.py index 155c2e7e90..e722606598 100644 --- a/flax/nnx/reprlib.py +++ b/flax/nnx/reprlib.py @@ -12,9 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import contextlib import dataclasses -import os -import sys import threading import typing as tp @@ -22,125 +21,22 @@ B = tp.TypeVar('B') -def supports_color() -> bool: - """ - Returns True if the running system's terminal supports color, and False otherwise. - """ - try: - from IPython import get_ipython - - ipython_available = get_ipython() is not None - except ImportError: - ipython_available = False - - supported_platform = sys.platform != 'win32' or 'ANSICON' in os.environ - is_a_tty = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty() - return (supported_platform and is_a_tty) or ipython_available - - -class Color(tp.NamedTuple): - TYPE: str - ATTRIBUTE: str - SEP: str - PAREN: str - COMMENT: str - INT: str - STRING: str - FLOAT: str - BOOL: str - NONE: str - END: str - - -NO_COLOR = Color( - TYPE='', - ATTRIBUTE='', - SEP='', - PAREN='', - COMMENT='', - INT='', - STRING='', - FLOAT='', - BOOL='', - NONE='', - END='', -) - - -# Use python vscode theme colors -if supports_color(): - COLOR = Color( - TYPE='\x1b[38;2;79;201;177m', - ATTRIBUTE='\033[38;2;156;220;254m', - SEP='\x1b[38;2;212;212;212m', - PAREN='\x1b[38;2;255;213;3m', - # COMMENT='\033[38;2;87;166;74m', - COMMENT='\033[38;2;105;105;105m', # Dark gray - INT='\x1b[38;2;182;207;169m', - STRING='\x1b[38;2;207;144;120m', - FLOAT='\x1b[38;2;182;207;169m', - BOOL='\x1b[38;2;86;156;214m', - NONE='\x1b[38;2;86;156;214m', - END='\x1b[0m', - ) -else: - COLOR = NO_COLOR - - @dataclasses.dataclass class ReprContext(threading.local): - current_color: Color = COLOR + indent_stack: tp.List[str] = dataclasses.field(default_factory=lambda: ['']) REPR_CONTEXT = ReprContext() -def colorized(x, /): - c = REPR_CONTEXT.current_color - if isinstance(x, list): - return f'{c.PAREN}[{c.END}{", ".join(map(lambda i: colorized(i), x))}{c.PAREN}]{c.END}' - elif isinstance(x, tuple): - if len(x) == 1: - return f'{c.PAREN}({c.END}{colorized(x[0])},{c.PAREN}){c.END}' - return f'{c.PAREN}({c.END}{", ".join(map(lambda i: colorized(i), x))}{c.PAREN}){c.END}' - elif isinstance(x, dict): - open, close = '{', '}' - return f'{c.PAREN}{open}{c.END}{", ".join(f"{c.STRING}{k!r}{c.END}: {colorized(v)}" for k, v in x.items())}{c.PAREN}{close}{c.END}' - elif isinstance(x, set): - open, close = '{', '}' - return f'{c.PAREN}{open}{c.END}{", ".join(map(lambda i: colorized(i), x))}{c.PAREN}{close}{c.END}' - elif isinstance(x, type): - return f'{c.TYPE}{x.__name__}{c.END}' - elif isinstance(x, bool): - return f'{c.BOOL}{x}{c.END}' - elif isinstance(x, int): - return f'{c.INT}{x}{c.END}' - elif isinstance(x, str): - return f'{c.STRING}{x!r}{c.END}' - elif isinstance(x, float): - return f'{c.FLOAT}{x}{c.END}' - elif x is None: - return f'{c.NONE}{x}{c.END}' - elif isinstance(x, Representable): - return get_repr(x) - else: - return repr(x) - - @dataclasses.dataclass class Object: type: tp.Union[str, type] start: str = '(' end: str = ')' - kv_sep: str = '=' - indent: str = ' ' + value_sep: str = '=' + elem_indent: str = ' ' empty_repr: str = '' - comment: str = '' - same_line: bool = False - - @property - def elem_sep(self): - return ', ' if self.same_line else ',\n' @dataclasses.dataclass @@ -149,8 +45,6 @@ class Attr: value: tp.Union[str, tp.Any] start: str = '' end: str = '' - use_raw_value: bool = False - use_raw_key: bool = False class Representable: @@ -160,96 +54,87 @@ def __nnx_repr__(self) -> tp.Iterator[tp.Union[Object, Attr]]: raise NotImplementedError def __repr__(self) -> str: - current_color = REPR_CONTEXT.current_color - REPR_CONTEXT.current_color = NO_COLOR - try: - return get_repr(self) - finally: - REPR_CONTEXT.current_color = current_color - - def __str__(self) -> str: return get_repr(self) +@contextlib.contextmanager +def add_indent(indent: str) -> tp.Iterator[None]: + REPR_CONTEXT.indent_stack.append(REPR_CONTEXT.indent_stack[-1] + indent) + + try: + yield + finally: + REPR_CONTEXT.indent_stack.pop() + + +def get_indent() -> str: + return REPR_CONTEXT.indent_stack[-1] + + def get_repr(obj: Representable) -> str: if not isinstance(obj, Representable): raise TypeError(f'Object {obj!r} is not representable') - c = REPR_CONTEXT.current_color iterator = obj.__nnx_repr__() config = next(iterator) - if not isinstance(config, Object): raise TypeError(f'First item must be Config, got {type(config).__name__}') - kv_sep = f'{c.SEP}{config.kv_sep}{c.END}' - def _repr_elem(elem: tp.Any) -> str: if not isinstance(elem, Attr): raise TypeError(f'Item must be Elem, got {type(elem).__name__}') - value_repr = elem.value if elem.use_raw_value else colorized(elem.value) - value_repr = value_repr.replace('\n', '\n' + config.indent) - key = elem.key if elem.use_raw_key else f'{c.ATTRIBUTE}{elem.key}{c.END}' - indent = '' if config.same_line else config.indent + value = elem.value if isinstance(elem.value, str) else repr(elem.value) - return f'{indent}{elem.start}{key}{kv_sep}{value_repr}{elem.end}' + value = value.replace('\n', '\n' + config.elem_indent) - elems = config.elem_sep.join(map(_repr_elem, iterator)) + return f'{config.elem_indent}{elem.start}{elem.key}{config.value_sep}{value}{elem.end}' + + with add_indent(config.elem_indent): + elems = ',\n'.join(map(_repr_elem, iterator)) if elems: - if config.same_line: - elems_repr = elems - comment = '' - else: - elems_repr = '\n' + elems + '\n' - comment = f'{c.COMMENT}{config.comment}{c.END}' + elems = '\n' + elems + '\n' else: - elems_repr = config.empty_repr - comment = '' + elems = config.empty_repr type_repr = ( config.type if isinstance(config.type, str) else config.type.__name__ ) - type_repr = f'{c.TYPE}{type_repr}{c.END}' if type_repr else '' - start = f'{c.PAREN}{config.start}{c.END}' if config.start else '' - end = f'{c.PAREN}{config.end}{c.END}' if config.end else '' - out = f'{type_repr}{start}{comment}{elems_repr}{end}' - return out + return f'{type_repr}{config.start}{elems}{config.end}' class MappingReprMixin(Representable): def __nnx_repr__(self): - yield Object(type='', kv_sep=': ', start='{', end='}') + yield Object(type='', value_sep=': ', start='{', end='}') + + for key, value in self.items(): + yield Attr(repr(key), value) + +class SequenceReprMixin(Representable): + def __nnx_repr__(self): + yield Object(type='', value_sep='', start='[', end=']') + + for value in self: + yield Attr('', value) - for key, value in self.items(): # type: ignore - yield Attr(colorized(key), value, use_raw_key=True) @dataclasses.dataclass(repr=False) class PrettyMapping(Representable): mapping: tp.Mapping def __nnx_repr__(self): - yield Object(type=type(self), kv_sep=': ', start='({', end='})') + yield Object(type='', value_sep=': ', start='{', end='}') for key, value in self.mapping.items(): - yield Attr(colorized(key), value, use_raw_key=True) - -@dataclasses.dataclass(repr=False) -class SequenceReprMixin(Representable): - def __nnx_repr__(self): - yield Object(type=type(self), kv_sep='', start='([', end='])') - - for value in self: # type: ignore - yield Attr('', value, use_raw_key=True) - + yield Attr(repr(key), value) @dataclasses.dataclass(repr=False) class PrettySequence(Representable): - sequence: tp.Sequence + list: tp.Sequence def __nnx_repr__(self): - yield Object(type=type(self), kv_sep='', start='([', end='])') + yield Object(type='', value_sep='', start='[', end=']') - for value in self.sequence: - yield Attr('', value, use_raw_key=True) \ No newline at end of file + for value in self.list: + yield Attr('', value) \ No newline at end of file diff --git a/flax/nnx/rnglib.py b/flax/nnx/rnglib.py index ab9817acaa..ea9353d313 100644 --- a/flax/nnx/rnglib.py +++ b/flax/nnx/rnglib.py @@ -13,7 +13,6 @@ # limitations under the License. from __future__ import annotations -import dataclasses import functools import typing as tp @@ -48,7 +47,6 @@ class RngKey(RngState): ... NotKey = filterlib.All(RngState, filterlib.Not(RngKey)) -@dataclasses.dataclass(repr=False) class RngStream(Object): def __init__( self, @@ -56,13 +54,12 @@ def __init__( key: jax.Array, count: jax.Array, ): + if not isinstance(key, jax.Array): + raise TypeError(f'key must be a jax.Array, got {type(key)}') + self.key = RngKey(key, tag=tag) self.count = RngCount(count, tag=tag) - def __post_init__(self): - if not isinstance(self.key, jax.Array): - raise TypeError(f'key must be a jax.Array, got {type(self.key)}') - def __call__(self) -> jax.Array: self.check_valid_context( lambda: 'Cannot call RngStream from a different trace level' @@ -80,7 +77,7 @@ def __call__(self) -> jax.Array: ] -class Rngs(Object, tp.Mapping[str, tp.Callable[[], jax.Array]]): +class Rngs(Object): """NNX rng container class. To instantiate the ``Rngs``, pass in an integer, specifying the starting seed. ``Rngs`` can have different "streams", allowing the user to generate different @@ -237,6 +234,10 @@ def __getstate__(self): def __setstate__(self, state): vars(self).update(state) + def items(self): + for name in self: + yield name, self[name] + class ForkStates(tp.NamedTuple): split_keys: State @@ -302,14 +303,12 @@ def split_rngs( *, splits: int | tuple[int, ...], only: filterlib.Filter = ..., - squeeze: bool = False, ) -> SplitBackups: ... @tp.overload def split_rngs( *, splits: int | tuple[int, ...], only: filterlib.Filter = ..., - squeeze: bool = False, ) -> tp.Callable[[F], F]: ... def split_rngs( node: tp.Any = MISSING, @@ -317,7 +316,6 @@ def split_rngs( *, splits: int | tuple[int, ...], only: filterlib.Filter = ..., - squeeze: bool = False, ) -> SplitBackups | tp.Callable[[F], F]: """Splits the (nested) Rng states of the given node. @@ -415,18 +413,13 @@ def split_rngs( def split_rngs_decorator(f: F) -> F: @functools.wraps(f) def split_rngs_wrapper(*args, **kwargs): - with split_rngs( - (args, kwargs), splits=splits, only=only, squeeze=squeeze - ): + with split_rngs((args, kwargs), splits=splits, only=only): return f(*args, **kwargs) return tp.cast(F, split_rngs_wrapper) return split_rngs_decorator # type: ignore[bad-return-type] - if squeeze and splits != 1: - raise ValueError('squeeze=True is only supported for splits=1') - predicate = filterlib.to_predicate(only) backups: list[StreamBackup] = [] for path, stream in graph.iter_graph(node): @@ -437,13 +430,8 @@ def split_rngs_wrapper(*args, **kwargs): ): key = stream() backups.append((stream, stream.key.value, stream.count.value)) - key = jax.random.split(key, splits) - if squeeze: - key = key[0] - stream.key.value = key - if squeeze: - counts_shape = stream.count.shape - elif isinstance(splits, int): + stream.key.value = jax.random.split(key, splits) + if isinstance(splits, int): counts_shape = (splits, *stream.count.shape) else: counts_shape = (*splits, *stream.count.shape) diff --git a/flax/nnx/statelib.py b/flax/nnx/statelib.py index 38cb3da759..2f6ebff02e 100644 --- a/flax/nnx/statelib.py +++ b/flax/nnx/statelib.py @@ -38,7 +38,7 @@ def __init__(self, state: State): self.state = state def __nnx_repr__(self): - yield reprlib.Object('', kv_sep=': ', start='{', end='}') + yield reprlib.Object('', value_sep=': ', start='{', end='}') for r in self.state.__nnx_repr__(): if isinstance(r, reprlib.Object): @@ -54,26 +54,43 @@ def __treescope_repr__(self, path, subtree_renderer): # Render as the dictionary itself at the same path. return subtree_renderer(children, path=path) -class FlatState(tp.Sequence[tuple[PathParts, V]], reprlib.SequenceReprMixin): +class FlatState(tp.Sequence[tuple[PathParts, V]], reprlib.Representable): + __slots__ = ('_keys', '_values') + _keys: tuple[PathParts, ...] _values: list[V] - def __init__(self, items: tp.Iterable[tuple[PathParts, V]]): + def __init__(self, items: tp.Iterable[tuple[PathParts, V]], /, *, sort: bool): keys, values = [], [] + if sort: + items = sorted(items) for key, value in items: keys.append(key) values.append(value) self._keys = tuple(keys) self._values = values - @property - def paths(self) -> tp.Sequence[PathParts]: + @staticmethod + def from_sorted_keys_values( + keys: tuple[PathParts, ...], values: list[V], / + ) -> FlatState[V]: + flat_state = object.__new__(FlatState) + flat_state._keys = keys + flat_state._values = values + return flat_state + + def get_keys(self) -> tuple[PathParts, ...]: return self._keys - @property - def leaves(self) -> tp.Sequence[V]: + def get_values(self) -> tp.List[V]: return self._values + def __nnx_repr__(self): + yield reprlib.Object(type='FlatState', value_sep='', start='([', end='])') + + for value in self: + yield reprlib.Attr('', value) + @tp.overload def __getitem__(self, index: int) -> tuple[PathParts, V]: ... @tp.overload @@ -83,7 +100,7 @@ def __getitem__( ) -> tuple[PathParts, V] | FlatState[V]: if isinstance(index, int): return self._keys[index], self._values[index] - return FlatState(zip(self._keys[index], self._values[index])) + return FlatState(zip(self._keys[index], self._values[index]), sort=False) def __len__(self) -> int: return len(self._keys) @@ -91,6 +108,91 @@ def __len__(self) -> int: def __iter__(self) -> tp.Iterator[tuple[PathParts, V]]: return iter(zip(self._keys, self._values)) + def to_nested_state(self) -> State[PathParts, V]: + return State.from_flat_path(self) + + @tp.overload + def split(self, first: filterlib.Filter, /) -> FlatState[V]: ... + + @tp.overload + def split( + self, + first: filterlib.Filter, + second: filterlib.Filter, + /, + *filters: filterlib.Filter, + ) -> tuple[FlatState[V], ...]: ... + + @tp.overload + def split( + self, /, *filters: filterlib.Filter + ) -> tp.Union[FlatState[V], tuple[FlatState[V], ...]]: ... + + def split( # type: ignore[misc] + self, first: filterlib.Filter, /, *filters: filterlib.Filter + ) -> tp.Union[FlatState[V], tuple[FlatState[V], ...]]: + filters = (first, *filters) + *flat_states_, rest = _split_state(self, *filters) + + if rest: + raise ValueError( + 'Non-exhaustive filters, got a non-empty remainder: ' + f'{rest}.\nUse `...` to match all remaining elements.' + ) + + flat_states: FlatState[V] | tuple[FlatState[V], ...] + if len(flat_states_) == 1: + flat_states = flat_states_[0] + else: + flat_states = tuple(flat_states_) + return flat_states # type: ignore + + @tp.overload + def filter(self, first: filterlib.Filter, /) -> FlatState[V]: ... + + @tp.overload + def filter( + self, + first: filterlib.Filter, + second: filterlib.Filter, + /, + *filters: filterlib.Filter, + ) -> tuple[FlatState[V], ...]: ... + + def filter( + self, + first: filterlib.Filter, + /, + *filters: filterlib.Filter, + ) -> tp.Union[FlatState[V], tuple[FlatState[V], ...]]: + *flat_states_, _rest = _split_state(self, first, *filters) + + assert len(flat_states_) == len(filters) + 1 + + flat_states: FlatState[V] | tuple[FlatState[V], ...] + if len(flat_states_) == 1: + flat_states = flat_states_[0] + else: + flat_states = tuple(flat_states_) + + return flat_states # type: ignore + + @staticmethod + def merge( + flat_state: tp.Iterable[tuple[PathParts, V]], + /, + *flat_states: tp.Iterable[tuple[PathParts, V]], + ) -> FlatState[V]: + if not flat_states: + if isinstance(flat_state, FlatState): + return flat_state + return FlatState(flat_state, sort=True) + flat_states = (flat_state, *flat_states) + + return FlatState( + (elem for flat_state in flat_states for elem in flat_state), sort=True + ) + def _flat_state_pytree_flatten(x: FlatState[V]): return x._values, x._keys @@ -181,7 +283,7 @@ def __len__(self) -> int: return len(self._mapping) def __nnx_repr__(self): - yield reprlib.Object(type(self), kv_sep=': ', start='({', end='})') + yield reprlib.Object(type(self), value_sep=': ', start='({', end='})') for k, v in self.items(): if isinstance(v, State): @@ -211,7 +313,7 @@ def map(self, f: tp.Callable[[tuple, V], V]) -> State[K, V]: return State.from_flat_path(result) def flat_state(self) -> FlatState[V]: - return FlatState(traversals.flatten_to_sequence(self._mapping)) + return FlatState(traversals.flatten_to_sequence(self._mapping), sort=True) @classmethod def from_flat_path( @@ -299,7 +401,8 @@ def split( # type: ignore[misc] One or more ``States`` equal to the number of filters passed. """ filters = (first, *filters) - *states_, rest = _split_state(self.flat_state(), *filters) + flat_states = _split_state(self.flat_state(), *filters) + *states_, rest = (state.to_nested_state() for state in flat_states) if rest: raise ValueError( @@ -364,7 +467,8 @@ def filter( Returns: One or more ``States`` equal to the number of filters passed. """ - *states_, _rest = _split_state(self.flat_state(), first, *filters) + flat_states = _split_state(self.flat_state(), first, *filters) + *states_, _rest = (state.to_nested_state() for state in flat_states) assert len(states_) == len(filters) + 1 @@ -464,7 +568,7 @@ def _state_unflatten( def _split_state( flat_state: FlatState[V], *filters: filterlib.Filter, -) -> tuple[State[PathParts, V], ...]: +) -> tuple[FlatState[V], ...]: for i, filter_ in enumerate(filters): if filter_ in (..., True) and i != len(filters) - 1: remaining_filters = filters[i + 1 :] @@ -490,7 +594,7 @@ def _split_state( # if we didn't break, set leaf to last state flat_states[-1].append((path, value)) # type: ignore[index] # mypy is wrong here? - return tuple(State.from_flat_path(flat_state) for flat_state in flat_states) + return tuple(FlatState(flat_state, sort=False) for flat_state in flat_states) def create_path_filters(state: State): diff --git a/flax/nnx/tracers.py b/flax/nnx/tracers.py index a7b72b1540..c53bbd5c4d 100644 --- a/flax/nnx/tracers.py +++ b/flax/nnx/tracers.py @@ -18,7 +18,7 @@ import jax import jax.core -from flax.nnx import reprlib, visualization +from flax.nnx import reprlib def current_jax_trace(): @@ -47,11 +47,12 @@ def __nnx_repr__(self): yield reprlib.Attr('jax_trace', self._jax_trace) def __treescope_repr__(self, path, subtree_renderer): - return visualization.render_object_constructor( - object_type=type(self), - attributes={'jax_trace': self._jax_trace}, - path=path, - subtree_renderer=subtree_renderer, + import treescope # type: ignore[import-not-found,import-untyped] + return treescope.repr_lib.render_object_constructor( + object_type=type(self), + attributes={'jax_trace': self._jax_trace}, + path=path, + subtree_renderer=subtree_renderer, ) def __eq__(self, other): diff --git a/flax/nnx/training/metrics.py b/flax/nnx/training/metrics.py index 4facf42787..2073787b0d 100644 --- a/flax/nnx/training/metrics.py +++ b/flax/nnx/training/metrics.py @@ -276,45 +276,45 @@ class MultiMetric(Metric): ... ) >>> metrics - MultiMetric( # MetricState: 4 (16 B) - accuracy=Accuracy( # MetricState: 2 (8 B) + MultiMetric( + accuracy=Accuracy( argname='values', - total=MetricState( # 1 (4 B) + total=MetricState( value=Array(0., dtype=float32) ), - count=MetricState( # 1 (4 B) + count=MetricState( value=Array(0, dtype=int32) ) ), - loss=Average( # MetricState: 2 (8 B) + loss=Average( argname='values', - total=MetricState( # 1 (4 B) + total=MetricState( value=Array(0., dtype=float32) ), - count=MetricState( # 1 (4 B) + count=MetricState( value=Array(0, dtype=int32) ) ) ) >>> metrics.accuracy - Accuracy( # MetricState: 2 (8 B) + Accuracy( argname='values', - total=MetricState( # 1 (4 B) + total=MetricState( value=Array(0., dtype=float32) ), - count=MetricState( # 1 (4 B) + count=MetricState( value=Array(0, dtype=int32) ) ) >>> metrics.loss - Average( # MetricState: 2 (8 B) + Average( argname='values', - total=MetricState( # 1 (4 B) + total=MetricState( value=Array(0., dtype=float32) ), - count=MetricState( # 1 (4 B) + count=MetricState( value=Array(0, dtype=int32) ) ) diff --git a/flax/nnx/transforms/autodiff.py b/flax/nnx/transforms/autodiff.py index 5ef0d183b7..24ca8c9d6d 100644 --- a/flax/nnx/transforms/autodiff.py +++ b/flax/nnx/transforms/autodiff.py @@ -64,24 +64,26 @@ class DiffState: class GradFn: f: tp.Callable[..., tp.Any] has_aux: bool + nondiff_states: deque[State | None] def __post_init__(self): functools.update_wrapper(self, self.f) def __call__(self, *pure_args): # rebuild diff_state from substates in args - nondiff_states: deque[State | None] = extract.get_broadcast_state('grad') def _grad_merge_fn( ctx: graph.MergeContext, path, prefix, value: extract.NodeStates ): - nondiff = nondiff_states.popleft() + nondiff = self.nondiff_states.popleft() if nondiff is None: return ctx.merge(value.graphdef, value.state) else: return ctx.merge(value.graphdef, value.state, nondiff) - args = extract.from_tree(pure_args, merge_fn=_grad_merge_fn, ctxtag='grad') + args = extract.from_tree( + pure_args, merge_fn=_grad_merge_fn, ctxtag='grad', is_inner=True + ) out = self.f(*args) @@ -129,15 +131,6 @@ def _grad_general( else DiffState(-1, variablelib.Param) ) - gradded_fn = transform( - GradFn(f, has_aux), - argnums=jax_argnums, - has_aux=True, - holomorphic=holomorphic, - allow_int=allow_int, - reduce_axes=reduce_axes, - ) - @graph.update_context('grad') def grad_wrapper(*args, **kwargs): args = resolve_kwargs(f, args, kwargs) @@ -160,8 +153,16 @@ def _grad_split_fn( args, prefix=arg_filters, split_fn=_grad_split_fn, ctxtag='grad' ) - with extract.broadcast_state('grad', nondiff_states): - fn_out = gradded_fn(*pure_args) + gradded_fn = transform( + GradFn(f, has_aux, nondiff_states), + argnums=jax_argnums, + has_aux=True, + holomorphic=holomorphic, + allow_int=allow_int, + reduce_axes=reduce_axes, + ) + + fn_out = gradded_fn(*pure_args) def process_grads(grads): return jax.tree.map( @@ -171,7 +172,7 @@ def process_grads(grads): ) def process_out(pure_out: A, /) -> A: - return extract.from_tree(pure_out, ctxtag='grad') + return extract.from_tree(pure_out, ctxtag='grad', is_inner=False) if return_value: # unpack value_and_grad output @@ -427,11 +428,11 @@ def _custom_vjp_split_fn( nondiff_argnums: tuple[int, ...] = struct.field(pytree_node=False) tangent_tree_node_args: tuple[tp.Any, ...] = struct.field(pytree_node=False) -def _extract_index_mappings(x, *, index_mappings: deque[graph.HashableMapping]): +def _extract_nodedefs(x, *, nodedefs: deque[graph.NodeDef]): if isinstance(x, graph.NodeDef): - assert x.index_mapping is not None - index_mappings.append(x.index_mapping) - return dataclasses.replace(x, index_mapping=None) + assert x.outer_index is not None + nodedefs.append(x) + return x.with_no_outer_index() return x @dataclasses.dataclass(eq=False) @@ -440,6 +441,7 @@ class CustomVjpFnWrapper: jax_nondiff_argnums: tuple[int, ...] ctxtag: str nondiff_states: list[extract.GraphDefState] + nodedefs: deque[graph.NodeDef] def __post_init__(self): functools.update_wrapper(self, self.f) @@ -452,6 +454,7 @@ def __call__(self, *pure_args): _custom_vjp_merge_fn, nondiff_states=nondiff_states ), ctxtag=self.ctxtag, + is_inner=True, ) out = self.f(*args) @@ -464,13 +467,10 @@ def __call__(self, *pure_args): pure_args_out, pure_out = extract.to_tree( (args_out, out), ctxtag=self.ctxtag ) - # remove index_mapping from NodeDef's but store them in global context - index_mappings: deque[graph.HashableMapping] = extract.get_broadcast_state( - self.ctxtag - ) + # remove outer_index from NodeDef's but store them in global context pure_args_out, pure_out = jax.tree.map( - functools.partial(_extract_index_mappings, index_mappings=index_mappings), + functools.partial(_extract_nodedefs, nodedefs=self.nodedefs), (pure_args_out, pure_out), is_leaf=lambda x: isinstance(x, graph.NodeDef), ) @@ -484,6 +484,7 @@ class FwdFn: nondiff_argnums: tuple[int, ...] ctxtag: str nondiff_states: list[extract.GraphDefState] + nodedefs: deque[graph.NodeDef] def __post_init__(self): functools.update_wrapper(self, self.fwd) @@ -503,6 +504,7 @@ def __call__(self, *pure_args): _custom_vjp_merge_fn, nondiff_states=nondiff_states ), ctxtag=self.ctxtag if update_context_active else None, + is_inner=True, ) out, residual = self.fwd(*args) @@ -519,14 +521,9 @@ def __call__(self, *pure_args): pure_residual = extract.to_tree(residual) if update_context_active: - # remove index_mapping from NodeDef's but store them in global context - index_mappings: deque[graph.HashableMapping] = ( - extract.get_broadcast_state(self.ctxtag) - ) + # remove outer_index from NodeDef's but store them in global context pure_args_out, pure_out = jax.tree.map( - functools.partial( - _extract_index_mappings, index_mappings=index_mappings - ), + functools.partial(_extract_nodedefs, nodedefs=self.nodedefs), (pure_args_out, pure_out), is_leaf=lambda x: isinstance(x, graph.NodeDef), ) @@ -544,7 +541,7 @@ def __post_init__(self): def __call__(self, *args): *nondiff, pure_residual, (pure_args_out_g, pure_out_g) = args - residual = extract.from_tree(pure_residual) + residual = extract.from_tree(pure_residual, is_inner=True) (pure_args_out_g, pure_out_g) = jax.tree.map( lambda x: x.state if isinstance(x, extract.NodeStates) else x, (pure_args_out_g, pure_out_g), @@ -632,40 +629,41 @@ def __call__( for i, x in enumerate(tree_node_args) if i not in self.jax_nondiff_argnums ) - index_mappings: deque[graph.HashableMapping] = deque() - with extract.broadcast_state(self.ctxtag, index_mappings): - if self.fwd is None or self.bwd is None or self.symbolic_zeros is None: - raise ValueError() - - custom_vjp_fn = jax.custom_vjp( - fun=CustomVjpFnWrapper( - f=self.fun, - jax_nondiff_argnums=self.jax_nondiff_argnums, - ctxtag=self.ctxtag, - nondiff_states=nondiff_states, - ), + nodedefs: deque[graph.NodeDef] = deque() + if self.fwd is None or self.bwd is None or self.symbolic_zeros is None: + raise ValueError() + + custom_vjp_fn = jax.custom_vjp( + fun=CustomVjpFnWrapper( + f=self.fun, + jax_nondiff_argnums=self.jax_nondiff_argnums, + ctxtag=self.ctxtag, + nondiff_states=nondiff_states, + nodedefs=nodedefs, + ), + nondiff_argnums=self.jax_nondiff_argnums, + ) + custom_vjp_fn.defvjp( + fwd=FwdFn( + fwd=self.fwd, nondiff_argnums=self.jax_nondiff_argnums, - ) - custom_vjp_fn.defvjp( - fwd=FwdFn( - fwd=self.fwd, - nondiff_argnums=self.jax_nondiff_argnums, - ctxtag=self.ctxtag, - nondiff_states=nondiff_states, - ), - bwd=BwdFn( - bwd=self.bwd, - tree_node_args=tree_node_args, - ), - symbolic_zeros=self.symbolic_zeros, - ) - pure_args_out, pure_out = custom_vjp_fn(*pure_args) + ctxtag=self.ctxtag, + nondiff_states=nondiff_states, + nodedefs=nodedefs, + ), + bwd=BwdFn( + bwd=self.bwd, + tree_node_args=tree_node_args, + ), + symbolic_zeros=self.symbolic_zeros, + ) + pure_args_out, pure_out = custom_vjp_fn(*pure_args) # insert index_mappings def _insert_index_mappings(x): if isinstance(x, graph.NodeDef): - index_mapping: graph.HashableMapping = index_mappings.popleft() - return dataclasses.replace(x, index_mapping=index_mapping) + nodedef: graph.NodeDef = nodedefs.popleft() + return nodedef return x pure_args_out, pure_out = jax.tree_util.tree_map( @@ -675,7 +673,7 @@ def _insert_index_mappings(x): ) args_out, out = extract.from_tree( - (pure_args_out, pure_out), ctxtag=self.ctxtag + (pure_args_out, pure_out), ctxtag=self.ctxtag, is_inner=False ) return out diff --git a/flax/nnx/transforms/compilation.py b/flax/nnx/transforms/compilation.py index e5ce20f8e3..0336da35b1 100644 --- a/flax/nnx/transforms/compilation.py +++ b/flax/nnx/transforms/compilation.py @@ -91,9 +91,15 @@ def __hash__(self): def _jit_split_fn(ctx: graph.SplitContext, path, prefix, x): if isinstance(prefix, StateSharding): return extract.NodeStates.from_split( - *ctx.split(x, *prefix.filters), metadata=prefix + *ctx.flatten(x, *prefix.filters), metadata=prefix ) - return extract.NodeStates.from_split(*ctx.split(x)) + return extract.NodeStates.from_split(*ctx.flatten(x, with_paths=False)) + + +def _jit_merge_fn(ctx: graph.MergeContext, path, prefix, leaf) -> tp.Any: + if not isinstance(leaf, extract.NodeStates): + raise ValueError(f'Expected TreeNode, got {type(leaf)} at path {path}') + return ctx.unflatten(leaf.graphdef, *leaf.states) @dataclasses.dataclass(eq=False) @@ -102,12 +108,18 @@ class JitFn: in_shardings: tp.Any out_shardings: tp.Any kwarg_shardings: tp.Any + ctxtag: tp.Hashable def __post_init__(self): functools.update_wrapper(self, self.f) def __call__(self, *pure_args, **pure_kwargs): - args, kwargs = extract.from_tree((pure_args, pure_kwargs), ctxtag='jit') + args, kwargs = extract.from_tree( + (pure_args, pure_kwargs), + merge_fn=_jit_merge_fn, + ctxtag=self.ctxtag, + is_inner=True, + ) out = self.f(*args, **kwargs) @@ -115,7 +127,7 @@ def __call__(self, *pure_args, **pure_kwargs): pure_args_out, pure_kwargs_out, pure_out = extract.to_tree( (args_out, kwargs_out, out), prefix=(self.in_shardings, self.kwarg_shardings, self.out_shardings), - ctxtag='jit', + ctxtag=self.ctxtag, split_fn=_jit_split_fn, ) @@ -317,8 +329,32 @@ def jit( out_shardings, ) + @functools.wraps(fun) + def jit_wrapper(*args, **kwargs): + # run dynamic_cache_context before update_context + with graph.dynamic_cache(jit_wrapper), graph.update_context(jit_wrapper): + pure_args, pure_kwargs = extract.to_tree( + (args, kwargs), + prefix=(in_shardings, kwarg_shardings) + if in_shardings is not None or kwarg_shardings is not None + else None, + split_fn=_jit_split_fn, + check_aliasing=in_shardings is not None or kwarg_shardings is not None, + ctxtag=jit_wrapper, + ) + pure_args_out, pure_kwargs_out, pure_out = jitted_fn( + *pure_args, **pure_kwargs + ) + _args_out, _kwargs_out, out = extract.from_tree( + (pure_args_out, pure_kwargs_out, pure_out), + merge_fn=_jit_merge_fn, + is_inner=False, + ctxtag=jit_wrapper, + ) + return out + jitted_fn = jax.jit( - JitFn(fun, in_shardings, out_shardings, kwarg_shardings), + JitFn(fun, in_shardings, out_shardings, kwarg_shardings, jit_wrapper), in_shardings=jax_in_shardings, out_shardings=(jax_in_shardings, kwarg_shardings, jax_out_shardings), # type: ignore static_argnums=static_argnums, @@ -332,24 +368,6 @@ def jit( abstracted_axes=abstracted_axes, ) - @functools.wraps(fun) - @graph.update_context('jit') - def jit_wrapper(*args, **kwargs): - pure_args, pure_kwargs = extract.to_tree( - (args, kwargs), - prefix=(in_shardings, kwarg_shardings), - split_fn=_jit_split_fn, - check_aliasing=in_shardings is not None, - ctxtag='jit', - ) - pure_args_out, pure_kwargs_out, pure_out = jitted_fn( - *pure_args, **pure_kwargs - ) - _args_out, _kwargs_out, out = extract.from_tree( - (pure_args_out, pure_kwargs_out, pure_out), ctxtag='jit' - ) - return out - jit_wrapper.inner = jitted_fn # type: ignore return jit_wrapper # type: ignore diff --git a/flax/nnx/transforms/general.py b/flax/nnx/transforms/general.py index fa82cd890a..553c3e8926 100644 --- a/flax/nnx/transforms/general.py +++ b/flax/nnx/transforms/general.py @@ -151,7 +151,9 @@ def split_inputs( def split_inputs_wrapper(*args): pure_args = extract.to_tree(args, ctxtag=ctxtag) pure_args_out, pure_out = f(*pure_args) - args_out, out = extract.from_tree((pure_args_out, pure_out), ctxtag=ctxtag) + args_out, out = extract.from_tree( + (pure_args_out, pure_out), ctxtag=ctxtag, is_inner=False + ) return out return split_inputs_wrapper # type: ignore @@ -192,7 +194,7 @@ def merge_inputs( @functools.wraps(f) def merge_inputs_wrapper(*pure_args): - args = extract.from_tree(pure_args, ctxtag=ctxtag) + args = extract.from_tree(pure_args, ctxtag=ctxtag, is_inner=True) out = f(*args) args_out = extract.clear_non_graph_nodes(args) pure_args_out, pure_out = extract.to_tree((args_out, out), ctxtag=ctxtag) diff --git a/flax/nnx/transforms/iteration.py b/flax/nnx/transforms/iteration.py index 994e582862..e379cf1b9c 100644 --- a/flax/nnx/transforms/iteration.py +++ b/flax/nnx/transforms/iteration.py @@ -165,7 +165,7 @@ def __call__(self, *pure_args: tuple[tp.Any, ...]): pure_args = _update_variable_sharding_metadata( pure_args, self.transform_metadata, spmd.remove_axis ) - args = extract.from_tree(pure_args, ctxtag='vmap') + args = extract.from_tree(pure_args, ctxtag='vmap', is_inner=True) out = self.f(*args) @@ -343,7 +343,9 @@ def vmap_wrapper(*args, **kwargs): args, prefix=in_axes, split_fn=_vmap_split_fn, ctxtag='vmap' ) pure_args_out, pure_out = vmapped_fn(*pure_args) - _args_out, out = extract.from_tree((pure_args_out, pure_out), ctxtag='vmap') + _args_out, out = extract.from_tree( + (pure_args_out, pure_out), ctxtag='vmap', is_inner=False + ) return out return vmap_wrapper # type: ignore @@ -369,7 +371,7 @@ def __call__(self, *pure_args: tuple[tp.Any, ...]): pure_args = _update_variable_sharding_metadata( pure_args, self.transform_metadata, spmd.remove_axis ) - args = extract.from_tree(pure_args, ctxtag='pmap') + args = extract.from_tree(pure_args, ctxtag='pmap', is_inner=True) out = self.f(*args) @@ -566,7 +568,9 @@ def vmap_wrapper(*args): args, prefix=in_axes, split_fn=_vmap_split_fn, ctxtag='pmap' ) pure_args_out, pure_out = pmapped_fn(*pure_args) - _args_out, out = extract.from_tree((pure_args_out, pure_out), ctxtag='pmap') + _args_out, out = extract.from_tree( + (pure_args_out, pure_out), ctxtag='pmap', is_inner=False + ) return out return vmap_wrapper # type: ignore @@ -648,21 +652,17 @@ def check_carry_same_references(key_path, arg, out): check_carry_same_references, carry_arg, carry_arg_out ) -def _extract_index_mappings( - pure_carry_arg_out, - carry_index_mappings: list[graph.HashableMapping[int, int]], - /, +def _extract_nodedefs( + pure_carry_arg_out, carry_nodedefs: list[graph.NodeDef], / ): def extract_index_mappings(x): if isinstance(x, extract.NodeStates) and isinstance( x._graphdef, graph.NodeDef ): - index_mapping = x._graphdef.index_mapping - assert index_mapping is not None - carry_index_mappings.append(index_mapping) - x = x.replace( - _graphdef=dataclasses.replace(x._graphdef, index_mapping=None) - ) + nodedef = x._graphdef + assert nodedef.outer_index is not None + carry_nodedefs.append(nodedef) + x = x.replace(_graphdef=nodedef.with_no_outer_index()) return x pure_carry_arg_out = jax.tree.map( @@ -673,19 +673,17 @@ def extract_index_mappings(x): return pure_carry_arg_out -def _insert_index_mappings( +def _insert_nodedefs( pure_carry_arg_out, - carry_index_mappings: deque[graph.HashableMapping[int, int]], + carry_nodedefs: deque[graph.NodeDef], /, ): def insert_index_mappings(x): if isinstance(x, extract.NodeStates) and isinstance( x._graphdef, graph.NodeDef ): - index_mapping = carry_index_mappings.popleft() - x = x.replace( - _graphdef=dataclasses.replace(x._graphdef, index_mapping=index_mapping) - ) + nodedef = carry_nodedefs.popleft() + x = x.replace(_graphdef=nodedef) return x pure_carry_arg_out = jax.tree.map( @@ -1017,6 +1015,7 @@ def __call__( is_leaf=lambda x: isinstance(x, (extract.NodeStates, Broadcasted)), map_non_graph_nodes=True, ctxtag='scan', + is_inner=True, ) assert not carry_deque and not broadcast_deque and not broadcast_arrays @@ -1096,10 +1095,8 @@ def __call__( # next we have to remove all the index_mappings from the NodeDefs # in the carry outputs because they are not present in the inputs - carry_index_mappings: list[graph.HashableMapping[int, int]] = [] - pure_carry_arg_out = _extract_index_mappings( - pure_carry_arg_out, carry_index_mappings - ) + carry_nodedefs: list[graph.NodeDef] = [] + pure_carry_arg_out = _extract_nodedefs(pure_carry_arg_out, carry_nodedefs) carry_arg_out = ( pure_carry_arg_out, @@ -1108,7 +1105,7 @@ def __call__( broadcast_arrays_out, ) scan_out = ( - graph.Static(tuple(carry_index_mappings)), + carry_nodedefs, pure_args_out, pure_out, ) @@ -1248,16 +1245,15 @@ def scan_wrapper(*args, **kwargs): broadcast_arrays_out, ) = carry_out ( - static_carry_index_mappings, + carry_nodedefs, pure_args_out, pure_out, ) = scan_out # next we have to insert all the index_mappings back into the NodeDefs # in the carry outputs - carry_index_mappings = deque(static_carry_index_mappings.value) - pure_carry_arg_out = _insert_index_mappings( - pure_carry_arg_out, carry_index_mappings + pure_carry_arg_out = _insert_nodedefs( + pure_carry_arg_out, deque(carry_nodedefs) ) # insert pure carry into pure_args_out @@ -1280,6 +1276,7 @@ def scan_wrapper(*args, **kwargs): is_leaf=lambda x: isinstance(x, (extract.NodeStates, Broadcasted)), map_non_graph_nodes=True, ctxtag='scan', + is_inner=False, ) # extract the carry from args_out @@ -1330,35 +1327,15 @@ def __call__(self, pure_val): def _add_fake_index_mapping(tree: tp.Any): global_index_mapping = {} # for the whole context, over all inputs - def per_node_state(ns: extract.NodeStates | tp.Any): - if not isinstance(ns, extract.NodeStates) or not isinstance( - ns._graphdef, graph.NodeDef + + def per_node_state(node_state: extract.NodeStates | tp.Any): + if not isinstance(node_state, extract.NodeStates) or not isinstance( + node_state._graphdef, graph.NodeDef ): - return ns - - def per_node_def(nd: graph.NodeDef | graph.NodeRef): - if nd.index >= 0: - global_index_mapping[nd.index] = nd.index - if isinstance(nd, graph.NodeRef): - return - - for attribute in nd.attributes: - if type(attribute) is graph.SubGraphAttribute: - per_node_def(attribute.value) - elif ( - type(attribute) is graph.LeafAttribute - and isinstance(attribute.value, (graph.VariableDef, graph.NodeRef)) - and attribute.value.index >= 0 - ): - global_index_mapping[attribute.value.index] = attribute.value.index - return - - per_node_def(ns._graphdef) + return node_state + return dataclasses.replace( - ns, - _graphdef=dataclasses.replace( - ns._graphdef, index_mapping=graph.HashableMapping(global_index_mapping) - ), + node_state, _graphdef=node_state._graphdef.with_same_outer_index() ) return jax.tree.map(per_node_state, tree, @@ -1366,16 +1343,18 @@ def per_node_def(nd: graph.NodeDef | graph.NodeRef): def _remove_index_mapping(tree: tp.Any): - '''Remove a fake index_mapping for the input to match that of the output.''' - def per_node_state(ns: extract.NodeStates | tp.Any): - if not isinstance(ns, extract.NodeStates) or not isinstance( - ns._graphdef, graph.NodeDef + """Remove a fake outer_index for the input to match that of the output.""" + + def per_node_state(node_state: extract.NodeStates | tp.Any): + if not isinstance(node_state, extract.NodeStates) or not isinstance( + node_state._graphdef, graph.NodeDef ): - return ns - assert isinstance(ns._graphdef, graph.NodeDef) - return dataclasses.replace(ns, _graphdef=dataclasses.replace( - ns._graphdef, index_mapping=None - )) + return node_state + assert isinstance(node_state._graphdef, graph.NodeDef) + node_state = dataclasses.replace( + node_state, _graphdef=node_state._graphdef.with_no_outer_index() + ) + return node_state return jax.tree.map(per_node_state, tree, is_leaf=lambda x: isinstance(x, extract.NodeStates)) @@ -1393,19 +1372,23 @@ def __call__(self, pure_val): # Removing the dummy index mapping being added outside of body function. pure_val_in = _remove_index_mapping(pure_val) - val = extract.from_tree(pure_val_in, ctxtag='while_loop_body') + val = extract.from_tree( + pure_val_in, ctxtag='while_loop_body', is_inner=True + ) out = self.f(val) pure_out = extract.to_tree(out, ctxtag='while_loop_body') try: jax.tree.map(lambda a, b: None, pure_val, pure_out) except ValueError as e: - msg = ("nnx.while_loop requires body function's input and output to " - "have the same reference and pytree structure, but they differ. " - "If the mismatch comes from `index_mapping` field, you might " - "have modified reference structure within the body function, " - "which is not allowed." - f"Detail of the mismatch: \n {str(e)}") + msg = ( + "nnx.while_loop requires body function's input and output to " + 'have the same reference and pytree structure, but they differ. ' + 'If the mismatch comes from `outer_index` field, you might ' + 'have modified reference structure within the body function, ' + 'which is not allowed.' + f'Detail of the mismatch: \n {str(e)}' + ) raise ValueError(msg) return pure_out @@ -1456,7 +1439,7 @@ def while_loop(cond_fun: tp.Callable[[T], tp.Any], WhileLoopBodyFn(body_fun), pure_init_val, ) - out = extract.from_tree(pure_out, ctxtag='while_loop') + out = extract.from_tree(pure_out, ctxtag='while_loop', is_inner=False) return out @@ -1472,19 +1455,21 @@ def __call__(self, i, pure_val): # Removing the dummy index mapping being added outside of body function. pure_val_in = _remove_index_mapping(pure_val) - val = extract.from_tree(pure_val_in, ctxtag='fori_loop_body') + val = extract.from_tree(pure_val_in, ctxtag='fori_loop_body', is_inner=True) out = self.f(i, val) pure_out = extract.to_tree(out, ctxtag='fori_loop_body') try: jax.tree.map(lambda a, b: None, pure_val, pure_out) except ValueError as e: - msg = ("nnx.fori_loop requires body function's input and output to " - "have the same reference and pytree structure, but they differ. " - "If the mismatch comes from `index_mapping` field, you might " - "have modified reference structure within the body function, " - "which is not allowed. " - f"Detail of the mismatch: \n {str(e)}") + msg = ( + "nnx.fori_loop requires body function's input and output to " + 'have the same reference and pytree structure, but they differ. ' + 'If the mismatch comes from `outer_index` field, you might ' + 'have modified reference structure within the body function, ' + 'which is not allowed. ' + f'Detail of the mismatch: \n {str(e)}' + ) raise ValueError(msg) return pure_out @@ -1545,5 +1530,5 @@ def fori_loop(lower: int, upper: int, pure_out = jax.lax.fori_loop(lower, upper, ForiLoopBodyFn(body_fun), pure_init_val, unroll=unroll) - out = extract.from_tree(pure_out, ctxtag='fori_loop') + out = extract.from_tree(pure_out, ctxtag='fori_loop', is_inner=False) return out diff --git a/flax/nnx/transforms/transforms.py b/flax/nnx/transforms/transforms.py index 8a83a026d4..3192b31aa7 100644 --- a/flax/nnx/transforms/transforms.py +++ b/flax/nnx/transforms/transforms.py @@ -160,7 +160,7 @@ def __post_init__(self): def __call__(self, *pure_args, **pure_kwargs): args, kwargs = extract.from_tree( - (pure_args, pure_kwargs), ctxtag='checkify' + (pure_args, pure_kwargs), ctxtag='checkify', is_inner=True ) out = self.f(*args, **kwargs) @@ -216,6 +216,7 @@ def jit_wrapper(*args, **kwargs): args_out, kwargs_out, out = extract.from_tree( (pure_args_out, pure_kwargs_out, pure_out), ctxtag='checkify', + is_inner=False, ) return error, out diff --git a/flax/nnx/variablelib.py b/flax/nnx/variablelib.py index b2c0660962..2b8a2af8ae 100644 --- a/flax/nnx/variablelib.py +++ b/flax/nnx/variablelib.py @@ -21,15 +21,10 @@ from typing import Any import jax -import treescope # type: ignore[import-untyped] from flax import errors -from flax.nnx import filterlib, reprlib, tracers, visualization -from flax.typing import ( - Missing, - PathParts, - value_stats, -) +from flax.nnx import filterlib, reprlib, tracers +from flax.typing import Missing, PathParts import jax.tree_util as jtu A = tp.TypeVar('A') @@ -47,7 +42,6 @@ VariableTypeCache: dict[str, tp.Type[Variable[tp.Any]]] = {} - @dataclasses.dataclass class VariableMetadata(tp.Generic[A]): raw_value: A @@ -125,6 +119,8 @@ class Variable(tp.Generic[A], reprlib.Representable): }) """ + __slots__ = ('raw_value', '_trace_state', '_var_metadata') + raw_value: A _trace_state: tracers.TraceState _var_metadata: dict[str, tp.Any] @@ -134,9 +130,8 @@ def __init__( value: tp.Union[A, VariableMetadata[A]], **metadata: tp.Any, ): - type_vars = vars(type(self)) - vars_self = vars(self) - vars_self['_trace_state'] = tracers.TraceState() + var_t = type(self) + object.__setattr__(self, '_trace_state', tracers.TraceState()) if isinstance(value, VariableMetadata): metadata.update(value.metadata) @@ -144,27 +139,30 @@ def __init__( object.__setattr__(self, 'raw_value', value) - if 'on_get_value' in type_vars and 'on_get_value' not in metadata: - metadata['get_value'] = getattr(type(self), 'on_get_value') + if hasattr(var_t, 'on_get_value') and 'on_get_value' not in metadata: + metadata['get_value'] = var_t.on_get_value - if 'on_set_value' in type_vars and 'on_set_value' not in metadata: - metadata['set_value'] = getattr(type(self), 'on_set_value') + if hasattr(var_t, 'on_set_value') and 'on_set_value' not in metadata: + metadata['set_value'] = var_t.on_set_value - if 'on_create_value' in type_vars and 'on_create_value' not in metadata: - metadata['create_value'] = getattr(type(self), 'on_create_value') + if hasattr(var_t, 'on_create_value') and 'on_create_value' not in metadata: + metadata['create_value'] = var_t.on_create_value - if 'on_add_axis' in type_vars and 'on_add_axis' not in metadata: - metadata['add_axis'] = getattr(type(self), 'on_add_axis') + if hasattr(var_t, 'on_add_axis') and 'on_add_axis' not in metadata: + metadata['add_axis'] = var_t.on_add_axis - if 'on_remove_axis' in type_vars and 'on_remove_axis' not in metadata: - metadata['remove_axis'] = getattr(type(self), 'on_remove_axis') + if hasattr(var_t, 'on_remove_axis') and 'on_remove_axis' not in metadata: + metadata['remove_axis'] = var_t.on_remove_axis - vars_self['_var_metadata'] = metadata + object.__setattr__(self, '_var_metadata', metadata) # run create_value hooks - vars_self['raw_value'] = self.create_value(self.raw_value) + object.__setattr__(self, 'raw_value', self.create_value(self.raw_value)) + + # def __hash__(self) -> int: + # return id(self) def __getattr__(self, name: str) -> tp.Any: - if name in vars(self)['_var_metadata']: + if name in object.__getattribute__(self, '_var_metadata'): return self._var_metadata[name] return getattr(self.value, name) @@ -220,9 +218,10 @@ def copy_from(self, other: Variable[A]) -> None: self._var_metadata.update(other.get_metadata()) def update_from_state(self, variable_state: VariableState[A]): - vars_self = vars(self) - vars_self['raw_value'] = variable_state.value - vars_self['_var_metadata'] = variable_state._var_metadata.copy() + object.__setattr__(self, 'raw_value', variable_state.value) + object.__setattr__( + self, '_var_metadata', variable_state._var_metadata.copy() + ) @property def value(self) -> A: @@ -239,7 +238,7 @@ def value(self, value: A): ) if 'on_set_value' in self._var_metadata: value = self._var_metadata['on_set_value'](self, value) - vars(self)['raw_value'] = value + object.__setattr__(self, 'raw_value', value) def create_value(self, value: A): if 'on_create_value' in self._var_metadata: @@ -254,9 +253,6 @@ def remove_axis(self, axis_index: AxisIndex, axis_name: AxisName | None): if 'on_remove_axis' in self._var_metadata: self._var_metadata['on_remove_axis'](self, axis_index, axis_name) - def __eq__(self, other: object) -> bool: - return type(self) is type(other) and vars(other) == vars(self) - @tp.overload def replace(self, value: B, **kwargs) -> Variable[B]: ... @@ -317,34 +313,20 @@ def to_state(self: Variable[A]) -> VariableState[A]: return VariableState(type(self), self.raw_value, **self._var_metadata) def __nnx_repr__(self): - stats = value_stats(self.value) - if stats: - comment = f' # {stats}' - else: - comment = '' - - yield reprlib.Object(type=type(self).__name__, comment=comment) + yield reprlib.Object(type=type(self)) yield reprlib.Attr('value', self.raw_value) for name, value in self._var_metadata.items(): yield reprlib.Attr(name, repr(value)) def __treescope_repr__(self, path, subtree_renderer): - size_bytes = value_stats(self.value) - if size_bytes: - stats_repr = f' # {size_bytes}' - first_line_annotation = treescope.rendering_parts.comment_color( - treescope.rendering_parts.text(f'{stats_repr}') - ) - else: - first_line_annotation = None + import treescope # type: ignore[import-not-found,import-untyped] children = {'value': self.raw_value, **self._var_metadata} - return visualization.render_object_constructor( + return treescope.repr_lib.render_object_constructor( object_type=type(self), attributes=children, path=path, subtree_renderer=subtree_renderer, - first_line_annotation=first_line_annotation, ) # hooks API @@ -369,10 +351,16 @@ def __jax_array__(self): # pickle support def __getstate__(self): - return vars(self).copy() + return { + 'raw_value': self.raw_value, + '_trace_state': self._trace_state, + '_var_metadata': self._var_metadata, + } def __setstate__(self, state): - vars(self).update(state) + object.__setattr__(self, 'raw_value', state['raw_value']) + object.__setattr__(self, '_trace_state', state['_trace_state']) + object.__setattr__(self, '_var_metadata', state['_var_metadata']) # -------------------------------------------- # proxy methods @@ -784,35 +772,22 @@ def __delattr__(self, name: str) -> None: del self._var_metadata[name] def __nnx_repr__(self): - stats = value_stats(self.value) - if stats: - comment = f' # {stats}' - else: - comment = '' - - yield reprlib.Object(type=type(self), comment=comment) - yield reprlib.Attr('type', self.type) + yield reprlib.Object(type=type(self)) + yield reprlib.Attr('type', self.type.__name__) yield reprlib.Attr('value', self.value) for name, value in self._var_metadata.items(): - yield reprlib.Attr(name, value) + yield reprlib.Attr(name, repr(value)) def __treescope_repr__(self, path, subtree_renderer): - size_bytes = value_stats(self.value) - if size_bytes: - stats_repr = f' # {size_bytes}' - first_line_annotation = treescope.rendering_parts.comment_color( - treescope.rendering_parts.text(f'{stats_repr}') - ) - else: - first_line_annotation = None + import treescope # type: ignore[import-not-found,import-untyped] + children = {'type': self.type, 'value': self.value, **self._var_metadata} - return visualization.render_object_constructor( + return treescope.repr_lib.render_object_constructor( object_type=type(self), attributes=children, path=path, subtree_renderer=subtree_renderer, - first_line_annotation=first_line_annotation, ) def replace(self, value: B) -> VariableState[B]: @@ -841,6 +816,7 @@ def remove_axis(self, axis_index: AxisIndex, axis_name: AxisName | None): if 'on_remove_axis' in self._var_metadata: self._var_metadata['on_remove_axis'](self, axis_index, axis_name) +GraphVariableState = VariableState[VariableState[tp.Any]] def _variable_state_flatten(x: VariableState[tp.Any], *, with_keys: bool): metadata = tuple(x.get_metadata().items()) @@ -944,7 +920,7 @@ def wrapper(*args): def split_flat_state( flat_state: tp.Iterable[tuple[PathParts, Variable | VariableState]], - filters: tp.Sequence[filterlib.Filter], + filters: tuple[filterlib.Filter, ...], ) -> tuple[list[tuple[PathParts, Variable | VariableState]], ...]: predicates = filterlib.filters_to_predicates(filters) # we have n + 1 states, where n is the number of predicates diff --git a/flax/nnx/visualization.py b/flax/nnx/visualization.py index 8c548d040c..d49eed7cf7 100644 --- a/flax/nnx/visualization.py +++ b/flax/nnx/visualization.py @@ -12,11 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import typing as tp - -import treescope # type: ignore[import-untyped] -from treescope import rendering_parts, renderers +import importlib.util +treescope_installed = importlib.util.find_spec('treescope') is not None try: from IPython import get_ipython @@ -31,112 +29,12 @@ def display(*args): If treescope is not installed or the code is not running in IPython, ``display`` will print the objects instead. """ - if not in_ipython: + if not treescope_installed or not in_ipython: for x in args: print(x) return + import treescope # type: ignore[import-not-found,import-untyped] + for x in args: treescope.display(x, ignore_exceptions=True, autovisualize=True) - - -def render_object_constructor( - object_type: type[tp.Any], - attributes: tp.Mapping[str, tp.Any], - path: str | None, - subtree_renderer: renderers.TreescopeSubtreeRenderer, - roundtrippable: bool = False, - color: str | None = None, - first_line_annotation: rendering_parts.RenderableTreePart | None = None, -) -> rendering_parts.Rendering: - """Renders an object in "constructor format", similar to a dataclass. - - This produces a rendering like `Foo(bar=1, baz=2)`, where Foo identifies the - type of the object, and bar and baz are the names of the attributes of the - object. It is a *requirement* that these are the actual attributes of the - object, which can be accessed via `obj.bar` or similar; otherwise, the - path renderings will break. - - This can be used from within a `__treescope_repr__` implementation via :: - - def __treescope_repr__(self, path, subtree_renderer): - return repr_lib.render_object_constructor( - object_type=type(self), - attributes=, - path=path, - subtree_renderer=subtree_renderer, - ) - - Args: - object_type: The type of the object. - attributes: The attributes of the object, which will be rendered as keyword - arguments to the constructor. - path: The path to the object. When `render_object_constructor` is called - from `__treescope_repr__`, this should come from the `path` argument to - `__treescope_repr__`. - subtree_renderer: The renderer to use to render subtrees. When - `render_object_constructor` is called from `__treescope_repr__`, this - should come from the `subtree_renderer` argument to `__treescope_repr__`. - roundtrippable: Whether evaluating the rendering as Python code will produce - an object that is equal to the original object. This implies that the - keyword arguments are actually the keyword arguments to the constructor, - and not some other attributes of the object. - color: The background color to use for the object rendering. If None, does - not use a background color. A utility for assigning a random color based - on a string key is given in `treescope.formatting_util`. - first_line_annotation: An annotation for the first line of the node when it - is expanded. - - Returns: - A rendering of the object, suitable for returning from `__treescope_repr__`. - """ - if roundtrippable: - constructor = rendering_parts.siblings( - rendering_parts.maybe_qualified_type_name(object_type), '(' - ) - closing_suffix = rendering_parts.text(')') - else: - constructor = rendering_parts.siblings( - rendering_parts.roundtrip_condition(roundtrip=rendering_parts.text('<')), - rendering_parts.maybe_qualified_type_name(object_type), - '(', - ) - closing_suffix = rendering_parts.siblings( - ')', - rendering_parts.roundtrip_condition(roundtrip=rendering_parts.text('>')), - ) - - children = [] - for i, (name, value) in enumerate(attributes.items()): - child_path = None if path is None else f'{path}.{name}' - - if i < len(attributes) - 1: - # Not the last child. Always show a comma, and add a space when - # collapsed. - comma_after = rendering_parts.siblings( - ',', - rendering_parts.fold_condition(collapsed=rendering_parts.text(' ')), - ) - else: - # Last child: only show the comma when the node is expanded. - comma_after = rendering_parts.fold_condition( - expanded=rendering_parts.text(',') - ) - - child_line = rendering_parts.build_full_line_with_annotations( - rendering_parts.siblings_with_annotations( - f'{name}=', - subtree_renderer(value, path=child_path), - ), - comma_after, - ) - children.append(child_line) - - return rendering_parts.build_foldable_tree_node_from_children( - prefix=constructor, - children=children, - suffix=closing_suffix, - path=path, - background_color=color, - first_line_annotation=first_line_annotation, - ) \ No newline at end of file diff --git a/flax/struct.py b/flax/struct.py index 6c18651aaa..4e8de0a7fe 100644 --- a/flax/struct.py +++ b/flax/struct.py @@ -123,7 +123,7 @@ class method that provides the smart constructor. """ # Support passing arguments to the decorator (e.g. @dataclass(kw_only=True)) if clz is None: - return functools.partial(dataclass, **kwargs) # type: ignore[bad-return-type] + return functools.partial(dataclass, **kwargs) # check if already a flax dataclass if '_flax_dataclass' in clz.__dict__: diff --git a/flax/typing.py b/flax/typing.py index 0ae990d95a..a630a3571e 100644 --- a/flax/typing.py +++ b/flax/typing.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import annotations from collections import deque from functools import partial @@ -27,8 +26,6 @@ from collections.abc import Callable, Hashable, Mapping, Sequence import jax -import jax.numpy as jnp -import numpy as np from flax.core import FrozenDict import dataclasses @@ -164,63 +161,3 @@ class Missing: MISSING = Missing() - - -def _bytes_repr(num_bytes): - count, units = ( - (f'{num_bytes / 1e9 :,.1f}', 'GB') - if num_bytes > 1e9 - else (f'{num_bytes / 1e6 :,.1f}', 'MB') - if num_bytes > 1e6 - else (f'{num_bytes / 1e3 :,.1f}', 'KB') - if num_bytes > 1e3 - else (f'{num_bytes:,}', 'B') - ) - - return f'{count} {units}' - - -class ShapeDtype(Protocol): - shape: Shape - dtype: Dtype - - -def has_shape_dtype(x: Any) -> TypeGuard[ShapeDtype]: - return hasattr(x, 'shape') and hasattr(x, 'dtype') - - -@dataclasses.dataclass(frozen=True, slots=True) -class SizeBytes: # type: ignore[misc] - size: int - bytes: int - - @staticmethod - def from_array(x: ShapeDtype) -> SizeBytes: - size = int(np.prod(x.shape)) - dtype: jnp.dtype - if isinstance(x.dtype, str): - dtype = jnp.dtype(x.dtype) - else: - dtype = x.dtype # type: ignore - bytes = size * dtype.itemsize # type: ignore - return SizeBytes(size, bytes) - - def __add__(self, other: SizeBytes) -> SizeBytes: - return SizeBytes(self.size + other.size, self.bytes + other.bytes) - - def __bool__(self) -> bool: - return bool(self.size) - - def __repr__(self) -> str: - bytes_repr = _bytes_repr(self.bytes) - return f'{self.size:,} ({bytes_repr})' - - -def value_stats(x): - leaves = jax.tree.leaves(x) - size_bytes = SizeBytes(0, 0) - for leaf in leaves: - if has_shape_dtype(leaf): - size_bytes += SizeBytes.from_array(leaf) - - return size_bytes \ No newline at end of file diff --git a/flaxlib_src/CMakeLists.txt b/flaxlib_src/CMakeLists.txt new file mode 100644 index 0000000000..a5a61b5b2a --- /dev/null +++ b/flaxlib_src/CMakeLists.txt @@ -0,0 +1,54 @@ +# Set the minimum CMake version and policies for highest tested version +cmake_minimum_required(VERSION 3.15...3.27) + +# Set up the project and ensure there is a working C++ compiler +project(flaxlib LANGUAGES CXX) + +# Warn if the user invokes CMake directly +if (NOT SKBUILD) + message(WARNING "\ + This CMake file is meant to be executed using 'scikit-build-core'. + Running it directly will almost certainly not produce the desired + result. If you are a user trying to install this package, use the + command below, which will install all necessary build dependencies, + compile the package in an isolated environment, and then install it. + ===================================================================== + $ pip install . + ===================================================================== + If you are a software developer, and this is your own package, then + it is usually much more efficient to install the build dependencies + in your environment once and use the following command that avoids + a costly creation of a new virtual environment at every compilation: + ===================================================================== + $ pip install nanobind scikit-build-core[pyproject] + $ pip install --no-build-isolation -ve . + ===================================================================== + You may optionally add -Ceditable.rebuild=true to auto-rebuild when + the package is imported. Otherwise, you need to rerun the above + after editing C++ files.") +endif() + +# Try to import all Python components potentially needed by nanobind +find_package(Python 3.8 + REQUIRED COMPONENTS Interpreter Development.Module + OPTIONAL_COMPONENTS Development.SABIModule) + +# Import nanobind through CMake's find_package mechanism +find_package(nanobind CONFIG REQUIRED) + +# We are now ready to compile the actual extension module +nanobind_add_module( + # Name of the extension + flaxlib_cpp + + # Target the stable ABI for Python 3.12+, which reduces + # the number of binary wheels that must be built. This + # does nothing on older Python versions + STABLE_ABI + + # Source code goes here + src/lib.cc +) + +# Install directive for scikit-build-core +install(TARGETS flaxlib_cpp LIBRARY DESTINATION flaxlib) \ No newline at end of file diff --git a/flaxlib_src/meson.build b/flaxlib_src/meson.build deleted file mode 100644 index 0d78d9436b..0000000000 --- a/flaxlib_src/meson.build +++ /dev/null @@ -1,14 +0,0 @@ -project( - 'flaxlib', - 'cpp', - version: '0.0.1', - default_options: ['cpp_std=c++17'], -) -py = import('python').find_installation() -nanobind_dep = dependency('nanobind', static: true) -py.extension_module( - 'flaxlib', - sources: ['src/lib.cc'], - dependencies: [nanobind_dep], - install: true, -) \ No newline at end of file diff --git a/flaxlib_src/pyproject.toml b/flaxlib_src/pyproject.toml index 0afc7699a5..fd6c0b61b4 100644 --- a/flaxlib_src/pyproject.toml +++ b/flaxlib_src/pyproject.toml @@ -1,17 +1,28 @@ [build-system] -requires = ['meson-python'] -build-backend = 'mesonpy' +requires = ["scikit-build-core >=0.4.3", "nanobind >=1.3.2"] +build-backend = "scikit_build_core.build" [project] name = "flaxlib" +version = "0.0.1" requires-python = ">=3.10" classifiers = [ "Programming Language :: C++", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ] -dynamic = ["version"] + [project.optional-dependencies] tests = [ "pytest", ] + +[tool.scikit-build] +# Protect the configuration against future changes in scikit-build-core +minimum-version = "0.4" + +# Setuptools-style build caching in a local directory +build-dir = "build/{wheel_tag}" + +# Build stable ABI wheels for CPython 3.12+ +wheel.py-api = "cp312" \ No newline at end of file diff --git a/flaxlib_src/flaxlib.pyi b/flaxlib_src/src/flaxlib/__init__.py similarity index 84% rename from flaxlib_src/flaxlib.pyi rename to flaxlib_src/src/flaxlib/__init__.py index 505fd3d0f0..f458417719 100644 --- a/flaxlib_src/flaxlib.pyi +++ b/flaxlib_src/src/flaxlib/__init__.py @@ -12,4 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -def sum_as_string(a: int, b: int) -> str: ... +from .flaxlib_cpp import RefMap as RefMap +from .flaxlib_cpp import _graph_fingerprint as _graph_fingerprint diff --git a/flaxlib_src/src/flaxlib/flaxlib_cpp.pyi b/flaxlib_src/src/flaxlib/flaxlib_cpp.pyi new file mode 100644 index 0000000000..03557efb9f --- /dev/null +++ b/flaxlib_src/src/flaxlib/flaxlib_cpp.pyi @@ -0,0 +1,25 @@ +# Copyright 2024 The Flax Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import typing as tp + +RefMap = tp.MutableMapping[tp.Any, int] + +def _graph_fingerprint( + node, + node_impl, + ref_index: RefMap, + new_ref_index: RefMap, + next_index: int, +) -> tuple[tuple[tp.Any, ...], int]: ... \ No newline at end of file diff --git a/flaxlib_src/src/lib.cc b/flaxlib_src/src/lib.cc index c714588118..c915727030 100644 --- a/flaxlib_src/src/lib.cc +++ b/flaxlib_src/src/lib.cc @@ -1,14 +1,298 @@ +// Copyright 2024 The Flax Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include -#include "nanobind/nanobind.h" -#include "nanobind/stl/string.h" +namespace nb = nanobind; +using namespace nb::literals; -namespace flaxlib { -std::string sum_as_string(int a, int b) { - return std::to_string(a + b); +// ----------------------------------- +// helper functions +// ----------------------------------- +intptr_t nb_id(const nb::object &obj) +{ + // Get the object ID + return reinterpret_cast(obj.ptr()); } -NB_MODULE(flaxlib, m) { - m.def("sum_as_string", &sum_as_string); +nb::tuple vector_to_tuple(const std::vector &vec) +{ + + if (vec.empty()) + { + return nb::tuple(); + } + else + { + return nb::tuple(nb::cast(vec)); + } } -} // namespace flaxlib \ No newline at end of file + +// 1. Hash function for nb::object +struct NbObjectHash +{ + std::size_t operator()(const nb::object &obj) const + { + return nb::hash(obj); + } +}; + +// 2. Equality function for nb::object (Important!) +struct NbObjectEqual +{ + bool operator()(const nb::object &a, const nb::object &b) const + { + return a.equal(b); + } +}; + +NB_MAKE_OPAQUE(std::unordered_map); + +namespace flaxlib +{ + //--------------------------------------------------------------- + // RefMap + //--------------------------------------------------------------- + + using RefMap = std::unordered_map; + + std::optional ref_map_get(RefMap &map, nb::object &key, std::optional default_value = std::nullopt) + { + auto it = map.find(key); + if (it != map.end()) + { + return it->second; + } + else + { + return std::nullopt; + } + } + + //--------------------------------------------------------------- + // NNXContext + //--------------------------------------------------------------- + + struct PythonContext + { + nb::object nnx; + nb::object graph; + nb::object jax; + nb::object np; + nb::object jax_Array; + nb::object np_ndarray; + nb::type_object GraphNodeImpl; + nb::type_object PytreeNodeImpl; + nb::type_object Object; + nb::type_object Variable; + nb::object get_node_impl; + + PythonContext() + { + nnx = nb::module_::import_("flax.nnx"); + graph = nb::module_::import_("flax.nnx.graph"); + jax = nb::module_::import_("jax"); + np = nb::module_::import_("numpy"); + jax_Array = jax.attr("Array"); + np_ndarray = np.attr("ndarray"); + GraphNodeImpl = graph.attr("GraphNodeImpl"); + PytreeNodeImpl = graph.attr("PytreeNodeImpl"); + Object = nnx.attr("Object"); + Variable = graph.attr("Variable"); + get_node_impl = graph.attr("get_node_impl"); + } + + ~PythonContext() + { + graph.release(); + jax.release(); + np.release(); + jax_Array.release(); + np_ndarray.release(); + GraphNodeImpl.release(); + PytreeNodeImpl.release(); + Variable.release(); + get_node_impl.release(); + } + }; + + static std::optional _python_context; + + PythonContext &get_python_context() + { + if (!_python_context) + { + _python_context.emplace(); + } + return *_python_context; + } + + //--------------------------------------------------------------- + // fingerprint + //--------------------------------------------------------------- + std::tuple _key_values_metadata( + PythonContext &ctx, + nb::object &node, + nb::object &node_impl) + { + if (nb::isinstance(node, ctx.Object)) + { + nb::dict nodes_dict = node.attr("__dict__"); + nb::handle object_state = nodes_dict["_object__state"]; + nb::del(nodes_dict["_object__state"]); + auto nodes = nodes_dict.items(); + nodes.sort(); + nodes_dict["_object__state"] = object_state; + auto metadata = nb::make_tuple(node.type(), object_state.attr("_initializing")); + return {nodes, metadata}; + } + else if (PyList_Check(node.ptr()) || PyTuple_Check(node.ptr())) + { + int i = 0; + nb::list values; + for (const auto &value : node) + { + values.append(nb::make_tuple(i, value)); + i += 1; + } + return {values, nb::none()}; + } + else + { + auto values_metadata = node_impl.attr("flatten")(node); + auto values = values_metadata[0]; + auto metadata = values_metadata[1]; + return {values, metadata}; + } + } + + nb::tuple _graph_fingerprint_recursive( + PythonContext &ctx, + nb::object &node, + nb::object &node_impl, + RefMap &ref_index, + RefMap &new_ref_index, + int &next_index) + { + bool is_pytree_node = node_impl.type().is(ctx.PytreeNodeImpl); + bool is_graph_node = node_impl.type().is(ctx.GraphNodeImpl); + + if (is_pytree_node) + { + // pass + } + else if (ref_index.find(node) != ref_index.end()) + { + return nb::make_tuple(nb_id(node), node.type(), ref_index[node]); + } + else if (new_ref_index.find(node) != new_ref_index.end()) + { + return nb::make_tuple(nb_id(node), node.type(), new_ref_index[node]); + } + + // only cache graph nodes + int index; + if (is_graph_node) + { + index = new_ref_index[node] = next_index; + next_index += 1; + } + else + { + index = -1; + } + + std::vector attributes; + + auto [values, metadata] = _key_values_metadata(ctx, node, node_impl); + + for (const auto &key_value : values) + { + nb::object key = key_value[0]; + nb::object value = key_value[1]; + auto value_node_impl = ctx.get_node_impl(value); + if (!value_node_impl.is_none()) + { + auto node_fp = _graph_fingerprint_recursive(ctx, value, value_node_impl, ref_index, new_ref_index, next_index); + attributes.push_back(nb::make_tuple(key, node_fp)); + } + else if (nb::isinstance(value, ctx.Variable)) + { + if (ref_index.find(value) != ref_index.end()) + { + attributes.push_back(nb::make_tuple(key, nb_id(value), value.type(), ref_index[value])); + } + else if (new_ref_index.find(value) != new_ref_index.end()) + { + attributes.push_back(nb::make_tuple(key, nb_id(value), value.type(), new_ref_index[value])); + } + else + { + auto variable_index = new_ref_index[value] = next_index; + next_index += 1; + auto var_meta = nb::tuple(value.attr("_var_metadata").attr("items")()); + attributes.push_back(nb::make_tuple(key, nb_id(value), value.type(), variable_index, var_meta)); + } + } + else // static attribute + { + if (nb::isinstance(value, ctx.jax_Array) || nb::isinstance(value, ctx.np_ndarray)) + { + auto repr = "Arrays leaves are not supported: " + nb::cast(nb::repr(value)); + } + attributes.push_back(nb::make_tuple(key, value)); + } + } + + auto node_fp = nb::make_tuple( + is_graph_node ? nb::cast(nb_id(node)) : nb::none(), + node_impl.attr("type"), + index, + vector_to_tuple(attributes), + metadata); + + return node_fp; + } + + nb::tuple _graph_fingerprint( + nb::object &node, + nb::object &node_impl, + RefMap &ref_index, + RefMap &new_ref_index, + int next_index) + { + auto ctx = get_python_context(); + auto node_fp = _graph_fingerprint_recursive(ctx, node, node_impl, ref_index, new_ref_index, next_index); + return nb::make_tuple(node_fp, next_index); + } + + NB_MODULE(flaxlib_cpp, m) + { + // Remove the conflicting binding + nb::bind_map(m, "RefMap") + .def("get", &ref_map_get, nb::arg("key").none(), nb::arg("default_value").none()); + m.def("_graph_fingerprint", &_graph_fingerprint); + } +} // namespace flaxlib \ No newline at end of file diff --git a/flaxlib_src/src/lib.rs b/flaxlib_src/src/lib.rs deleted file mode 100644 index cadab2ef22..0000000000 --- a/flaxlib_src/src/lib.rs +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2024 The Flax Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use pyo3::prelude::*; - -/// Formats the sum of two numbers as string. -#[pyfunction] -fn sum_as_string(a: usize, b: usize) -> PyResult { - Ok((a + b).to_string()) -} - -/// A Python module implemented in Rust. -#[pymodule] -fn flaxlib(_py: Python, m: &Bound) -> PyResult<()> { - m.add_function(wrap_pyfunction!(sum_as_string, m)?)?; - Ok(()) -} diff --git a/pyproject.toml b/pyproject.toml index f7a890fad0..318a7637ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ "rich>=11.1", "typing_extensions>=4.2", "PyYAML>=5.4.1", - "treescope>=0.1.7", + "treescope>=0.1.2", ] classifiers = [ "Development Status :: 3 - Alpha", @@ -229,3 +229,9 @@ quote-style = "single" [tool.uv] # Ignore uv.lock and always upgrade the package to the latest upgrade-package = ["jax", "jaxlib", "orbax-checkpoint"] + +[dependency-groups] +dev = [ + "nanobind>=2.4.0", + "scikit-build-core[pyproject]>=0.10.7", +] diff --git a/tests/jax_utils_test.py b/tests/jax_utils_test.py index d54262413b..c9cd9b3095 100644 --- a/tests/jax_utils_test.py +++ b/tests/jax_utils_test.py @@ -15,8 +15,6 @@ """Tests for flax.jax_utils.""" from functools import partial -import os -import re from absl.testing import absltest from absl.testing import parameterized @@ -28,21 +26,9 @@ NDEV = 4 -_xla_device_count_flag_regexp = ( - r'[-]{0,2}xla_force_host_platform_device_count=(\d+)?(\s|$)' -) - - -def set_n_cpu_devices(n: int): - xla_flags = os.getenv('XLA_FLAGS', '') - xla_flags = re.sub(_xla_device_count_flag_regexp, '', xla_flags) - os.environ['XLA_FLAGS'] = ' '.join( - [f'--xla_force_host_platform_device_count={n}'] + xla_flags.split() - ) - def setUpModule(): - set_n_cpu_devices(NDEV) + chex.set_n_cpu_devices(NDEV) class PadShardUnpadTest(chex.TestCase): diff --git a/tests/nnx/bridge/wrappers_test.py b/tests/nnx/bridge/wrappers_test.py index 5b65603a24..b353dd4925 100644 --- a/tests/nnx/bridge/wrappers_test.py +++ b/tests/nnx/bridge/wrappers_test.py @@ -228,7 +228,9 @@ def test_nnx_to_linen(self): assert y.shape == (1, 64) np.testing.assert_allclose(y, x @ variables['params']['kernel']) assert 'nnx' in variables - assert isinstance(variables['nnx']['graphdef'], nnx.GraphDef) + assert isinstance( + variables['nnx']['graphdef'], nnx.graph.NodeDef | nnx.graph.NodeRef + ) def test_nnx_to_linen_multiple_rngs(self): class NNXInner(nnx.Module): diff --git a/tests/nnx/graph_utils_test.py b/tests/nnx/graph_utils_test.py index a7bbf178cb..397198ae41 100644 --- a/tests/nnx/graph_utils_test.py +++ b/tests/nnx/graph_utils_test.py @@ -64,10 +64,26 @@ def test_flatten(self): g = [a, 3, a, nnx.Param(4)] refmap = nnx.graph.RefMap() - graphdef, state = nnx.graph.flatten(g, ref_index=refmap) + graphdef, flat_state = nnx.graph.flatten(g, ref_index=refmap) - state[0]['b'].raw_value = 2 - state[3].raw_value = 4 + assert flat_state[0][1].value == 2 + assert flat_state[1][1].value == 4 + + assert len(refmap) == 2 + assert a['b'] in refmap + assert g[3] in refmap + + def test_flatten_no_paths(self): + a = {'a': 1, 'b': nnx.Param(2)} + g = [a, 3, a, nnx.Param(4)] + + refmap = nnx.graph.RefMap() + graphdef, flat_state = nnx.graph.flatten( + g, ref_index=refmap, with_paths=False + ) + + assert flat_state[0] == 2 + assert flat_state[1] == 4 assert len(refmap) == 2 assert a['b'] in refmap @@ -108,9 +124,40 @@ def test_unflatten_empty(self): graphdef, state = nnx.split(g) - with self.assertRaisesRegex(ValueError, 'Expected key'): + with self.assertRaisesRegex( + ValueError, 'Not enough leaves to unflatten the graph' + ): nnx.graph.unflatten(graphdef, nnx.State({})) + def test_unflatten_return_variables(self): + a = Dict({'a': 1, 'b': nnx.Param(2)}) + g = List([a, 3, a, nnx.Param(4)]) + + graphdef, state = nnx.graph.flatten( + g, with_paths=False, return_variables=True + ) + + self.assertLen(state, 2) + self.assertIsInstance(state, list) + self.assertIsInstance(state[0], nnx.Param) + self.assertIsInstance(state[1], nnx.Param) + + def test_clone_with_same_variables(self): + a = Dict({'a': 1, 'b': nnx.Param(2)}) + g = List([a, 3, a, nnx.Param(4)]) + + graphdef, state = nnx.graph.flatten( + g, with_paths=False, return_variables=True + ) + + g2 = nnx.graph.unflatten(graphdef, state) + + self.assertIsNot(g, g2) + self.assertIsNot(g[0], g2[0]) + self.assertIsNot(g[2], g2[2]) + self.assertIs(g[0]['b'], g2[0]['b']) + self.assertIs(g[3], g2[3]) + def test_update_dynamic(self): a = {'a': 1, 'b': nnx.Param(2)} g = [a, 3, a, nnx.Param(4)] @@ -303,7 +350,7 @@ def __init__(self): assert 'tree' in state assert 'a' in state.tree - assert graphdef.attributes[0].value.type is nnx.graph.GenericPytree + assert graphdef.attributes[0][1].type is nnx.graph.GenericPytree m2 = nnx.merge(graphdef, state) @@ -329,26 +376,28 @@ def f(m: Foo): ref_out_idx_out = nnx.graph.RefMap() graphdef: nnx.graph.GraphDef[Foo] graphdef, state = nnx.graph.flatten(m, ref_index=ref_out_idx_out) + state = state.to_nested_state() @partial(jax.jit, static_argnums=(0,)) def f_pure(graphdef: nnx.graph.GraphDef[Foo], state): idx_out_ref_in: dict[int, Any] = {} m = nnx.graph.unflatten(graphdef, state, index_ref=idx_out_ref_in) + ref_in_idx_out = nnx.graph.RefMap( + {v: k for k, v in idx_out_ref_in.items()} + ) f(m) - ref_in_idx_in = nnx.graph.RefMap[Any, int]() - graphdef, state = nnx.graph.flatten(m, ref_index=ref_in_idx_in) - idx_out_idx_in = nnx.graph.compose_mapping(idx_out_ref_in, ref_in_idx_in) - static_out = nnx.graph.Static((graphdef, idx_out_idx_in)) - return state, static_out - - static_out: nnx.graph.Static - state, static_out = f_pure(graphdef, state) - idx_out_idx_in: dict[int, int] - graphdef, idx_out_idx_in = static_out.value - idx_in_ref_out = nnx.graph.compose_mapping_reversed( - ref_out_idx_out, idx_out_idx_in + ref_in_idx_in = nnx.graph.RefMap() + graphdef, state = nnx.graph.flatten( + m, ref_index=ref_in_idx_in, ref_outer_index=ref_in_idx_out + ) + state = state.to_nested_state() + return state, graphdef + + state, graphdef_out = f_pure(graphdef, state) + idx_out_ref_out = {v: k for k, v in ref_out_idx_out.items()} + m2 = nnx.graph.unflatten( + graphdef_out, state, outer_index_outer_ref=idx_out_ref_out ) - m2 = nnx.graph.unflatten(graphdef, state, index_ref_cache=idx_in_ref_out) assert m2 is m assert m2.a is b assert m2.b is a @@ -366,29 +415,31 @@ def f(m: Foo): a = m.a b = m.b - ref_out_idx_out = nnx.graph.RefMap[Any, int]() + ref_out_idx_out = nnx.graph.RefMap() graphdef: nnx.graph.GraphDef[Foo] graphdef, state = nnx.graph.flatten(m, ref_index=ref_out_idx_out) + idx_out_ref_out = {v: k for k, v in ref_out_idx_out.items()} + state = state.to_nested_state() @partial(jax.jit, static_argnums=(0,)) def f_pure(graphdef: nnx.graph.GraphDef[Foo], state): idx_out_ref_in: dict[int, Any] = {} m = nnx.graph.unflatten(graphdef, state, index_ref=idx_out_ref_in) + ref_in_idx_out = nnx.graph.RefMap( + {v: k for k, v in idx_out_ref_in.items()} + ) f(m) - ref_in_idx_in = nnx.graph.RefMap[Any, int]() - graphdef, state = nnx.graph.flatten(m, ref_index=ref_in_idx_in) - idx_out_idx_in = nnx.graph.compose_mapping(idx_out_ref_in, ref_in_idx_in) - static_out = nnx.graph.Static((graphdef, idx_out_idx_in)) - return state, static_out - - static_out: nnx.graph.Static - state, static_out = f_pure(graphdef, state) - idx_out_idx_in: dict[int, int] - graphdef, idx_out_idx_in = static_out.value - idx_in_ref_out = nnx.graph.compose_mapping_reversed( - ref_out_idx_out, idx_out_idx_in + ref_in_idx_in = nnx.graph.RefMap() + graphdef, state = nnx.graph.flatten( + m, ref_index=ref_in_idx_in, ref_outer_index=ref_in_idx_out + ) + state = state.to_nested_state() + return state, graphdef + + state, graphdef = f_pure(graphdef, state) + m2 = nnx.graph.unflatten( + graphdef, state, outer_index_outer_ref=idx_out_ref_out ) - m2 = nnx.graph.unflatten(graphdef, state, index_ref_cache=idx_in_ref_out) assert m2 is m assert m2.a is b assert m2.b is a @@ -406,26 +457,28 @@ def f(m: Foo): ref_out_idx_out = nnx.graph.RefMap() graphdef: nnx.graph.GraphDef[Foo] graphdef, state = nnx.graph.flatten(m, ref_index=ref_out_idx_out) + idx_out_ref_out = {v: k for k, v in ref_out_idx_out.items()} + state = state.to_nested_state() @partial(jax.jit, static_argnums=(0,)) def f_pure(graphdef: nnx.graph.GraphDef[Foo], state): idx_out_ref_in: dict[int, Any] = {} m = nnx.graph.unflatten(graphdef, state, index_ref=idx_out_ref_in) + ref_in_idx_out = nnx.graph.RefMap( + {v: k for k, v in idx_out_ref_in.items()} + ) f(m) - ref_in_idx_in = nnx.graph.RefMap[Any, int]() - graphdef, state = nnx.graph.flatten(m, ref_index=ref_in_idx_in) - idx_out_idx_in = nnx.graph.compose_mapping(idx_out_ref_in, ref_in_idx_in) - static_out = nnx.graph.Static((graphdef, idx_out_idx_in)) - return state, static_out - - static_out: nnx.graph.Static - state, static_out = f_pure(graphdef, state) - idx_out_idx_in: dict[int, int] - graphdef, idx_out_idx_in = static_out.value - idx_in_ref_out = nnx.graph.compose_mapping_reversed( - ref_out_idx_out, idx_out_idx_in + ref_in_idx_in = nnx.graph.RefMap() + graphdef, state = nnx.graph.flatten( + m, ref_index=ref_in_idx_in, ref_outer_index=ref_in_idx_out + ) + state = state.to_nested_state() + return state, graphdef + + state, graphdef_out = f_pure(graphdef, state) + m2 = nnx.graph.unflatten( + graphdef_out, state, outer_index_outer_ref=idx_out_ref_out ) - m2 = nnx.graph.unflatten(graphdef, state, index_ref_cache=idx_in_ref_out) assert m2 is m assert m2.ref is m2 @@ -582,7 +635,7 @@ def __init__(self): @jax.jit def f(graphdef1, state1, graphdef2, state2): - with nnx.graph.merge_context(ctxtag) as ctx: + with nnx.graph.merge_context(True, ctxtag) as ctx: m1 = ctx.merge(graphdef1, state1) m2 = ctx.merge(graphdef2, state2) @@ -603,7 +656,7 @@ def f(graphdef1, state1, graphdef2, state2): graphdef1, state1, graphdef2, state2 ) - with nnx.graph.merge_context(ctxtag) as ctx: + with nnx.graph.merge_context(False, ctxtag) as ctx: m1_out = ctx.merge(graphdef1, state1) m2_out = ctx.merge(graphdef2, state2) @@ -671,7 +724,7 @@ def __init__(self): @jax.jit def f(pure_tree): - impure_tree2 = nnx.from_tree(pure_tree, ctxtag=ctxtag) + impure_tree2 = nnx.from_tree(pure_tree, ctxtag=ctxtag, is_inner=True) m1_out = impure_tree2[0] m2_out = impure_tree2[2]['b'] @@ -700,7 +753,7 @@ def f(pure_tree): pure_tree2 = f(pure_tree) - impure_tree2 = nnx.from_tree(pure_tree2, ctxtag=ctxtag) + impure_tree2 = nnx.from_tree(pure_tree2, ctxtag=ctxtag, is_inner=False) m1_out = impure_tree2[0] m2_out = impure_tree2[2]['b'] @@ -762,7 +815,7 @@ def split_fn(ctx: nnx.SplitContext, path, prefix, x): @partial(jax.vmap, in_axes=jax_in_axes, out_axes=(jax_in_axes, out_axes)) def f(*pure_args): - args = nnx.from_tree(pure_args, ctxtag=ctxtag) + args = nnx.from_tree(pure_args, ctxtag=ctxtag, is_inner=True) y = 0 @@ -785,7 +838,9 @@ def f(*pure_args): pure_args_out, y = f(*pure_args) - args_out, y = nnx.from_tree((pure_args_out, y), ctxtag=ctxtag) + args_out, y = nnx.from_tree( + (pure_args_out, y), ctxtag=ctxtag, is_inner=False + ) self.assertEqual(y.shape, (5,)) self.assertGreater(y.sum(), 5) @@ -793,6 +848,44 @@ def f(*pure_args): self.assertIs(m1, args_out[2]['b']) self.assertIs(m2, args_out[1]) + def test_fingerprint_basic(self): + m = nnx.Linear(2, 3, rngs=nnx.Rngs(0)) + fp1 = nnx.graph.fingerprint(m) + fp2 = nnx.graph.fingerprint(m) + + self.assertEqual(fp1, fp2) + self.assertTrue(nnx.graph.check_fingerprint(m, fp1)) + self.assertTrue(nnx.graph.check_fingerprint(m, fp2)) + + def test_fingerprint_variable_id_sensitive(self): + m1 = nnx.Linear(2, 3, rngs=nnx.Rngs(0)) + fp1 = nnx.graph.fingerprint(m1) + + m2 = nnx.Linear(2, 3, rngs=nnx.Rngs(0)) + fp2 = nnx.graph.fingerprint(m2) + + self.assertNotEqual(fp1, fp2) + self.assertTrue(nnx.graph.check_fingerprint(m1, fp1)) + self.assertTrue(nnx.graph.check_fingerprint(m2, fp2)) + self.assertFalse(nnx.graph.check_fingerprint(m1, fp2)) + self.assertFalse(nnx.graph.check_fingerprint(m2, fp1)) + + def test_fingerprint_module_id_insensitive(self): + m1 = nnx.Linear(2, 3, rngs=nnx.Rngs(0)) + m2 = nnx.Linear(2, 3, rngs=nnx.Rngs(0)) + + m1.kernel = m2.kernel + m1.bias = m2.bias + + fp1 = nnx.graph.fingerprint(m1) + fp2 = nnx.graph.fingerprint(m2) + + self.assertNotEqual(fp1, fp2) + self.assertTrue(nnx.graph.check_fingerprint(m1, fp1)) + self.assertTrue(nnx.graph.check_fingerprint(m2, fp2)) + self.assertFalse(nnx.graph.check_fingerprint(m1, fp2)) + self.assertFalse(nnx.graph.check_fingerprint(m2, fp1)) + class SimpleModule(nnx.Module): pass diff --git a/tests/nnx/module_test.py b/tests/nnx/module_test.py index 64928f46b8..d5a89b0f3b 100644 --- a/tests/nnx/module_test.py +++ b/tests/nnx/module_test.py @@ -25,7 +25,6 @@ import jax.numpy as jnp import numpy as np - A = TypeVar('A') class List(nnx.Module): @@ -262,13 +261,13 @@ def test_clone(self): m2 = nnx.clone(m) assert m is not m2 - assert m2.a[0] == m2.b.c - assert m2.a[1] == m2.b.d + assert m2.a[0].value == m2.b.c.value + assert m2.a[1].value == m2.b.d.value - assert m.a[0] == m2.a[0] - assert m.a[1] == m2.a[1] - assert m.b.c == m2.b.c - assert m.b.d == m2.b.d + assert m.a[0].value == m2.a[0].value + assert m.a[1].value == m2.a[1].value + assert m.b.c.value == m2.b.c.value + assert m.b.d.value == m2.b.d.value def test_sow_basic(self): class Foo(nnx.Module): @@ -465,7 +464,7 @@ def __init__(self) -> None: m1 = Foo() m2 = deepcopy(m1) - assert m1.a == m2.a + assert m1.a.value == m2.a.value assert vars(m1)['a'] is not vars(m2)['a'] assert m1.b is not m2.b assert m1.c is not m2.c @@ -551,46 +550,6 @@ def __call__(self, x): y2 = model(jnp.ones((5, 2))) np.testing.assert_allclose(y1, y2) - def test_repr(self): - class Block(nnx.Module): - def __init__(self, din, dout, rngs: nnx.Rngs): - self.linear = nnx.Linear(din, dout, rngs=rngs) - self.bn = nnx.BatchNorm(dout, rngs=rngs) - self.dropout = nnx.Dropout(0.2, rngs=rngs) - - def __call__(self, x): - return nnx.relu(self.dropout(self.bn(self.linear(x)))) - - class Foo(nnx.Module): - def __init__(self, rngs: nnx.Rngs): - self.block1 = Block(32, 128, rngs=rngs) - self.block2 = Block(128, 10, rngs=rngs) - - def __call__(self, x): - return self.block2(self.block1(x)) - - obj = Foo(nnx.Rngs(0)) - - leaves = nnx.state(obj).flat_state().leaves - - expected_total = sum(int(np.prod(x.value.shape)) for x in leaves) - expected_total_params = sum( - int(np.prod(x.value.shape)) for x in leaves if x.type is nnx.Param - ) - expected_total_batch_stats = sum( - int(np.prod(x.value.shape)) for x in leaves if x.type is nnx.BatchStat - ) - expected_total_rng_states = sum( - int(np.prod(x.value.shape)) for x in leaves if x.type is nnx.RngState - ) - - foo_repr = repr(obj).replace(',', '').splitlines() - - self.assertIn(str(expected_total), foo_repr[0]) - self.assertIn(str(expected_total_params), foo_repr[0]) - self.assertIn(str(expected_total_batch_stats), foo_repr[0]) - self.assertIn(str(expected_total_rng_states), foo_repr[0]) - class TestModulePytree: def test_tree_map(self): @@ -639,6 +598,9 @@ class Foo(nnx.Module): e: nnx.Variable[int] f: int + def __hash__(self): + return id(self) + m = Foo( a=1, # graphdef b=nnx.Variable(2), # node @@ -717,7 +679,7 @@ def __call__(self, x, *, rngs: nnx.Rngs): graphdef, state = nnx.split(foo) - assert isinstance(graphdef, nnx.GraphDef) + assert isinstance(graphdef, nnx.graph.NodeDef | nnx.graph.NodeRef) assert isinstance(state, nnx.State) assert issubclass(state.w.type, nnx.Param) assert issubclass(state.c.type, nnx.Variable) diff --git a/tests/nnx/nn/recurrent_test.py b/tests/nnx/nn/recurrent_test.py index 0723a516a9..b724b69d7b 100644 --- a/tests/nnx/nn/recurrent_test.py +++ b/tests/nnx/nn/recurrent_test.py @@ -23,622 +23,521 @@ from absl.testing import absltest - class TestLSTMCell(absltest.TestCase): - def test_basic(self): - module = nnx.LSTMCell( - in_features=3, - hidden_features=4, - rngs=nnx.Rngs(0), - ) - x = jnp.ones((2, 3)) - carry = module.initialize_carry(x.shape, module.rngs) - new_carry, y = module(carry, x) - self.assertEqual(y.shape, (2, 4)) - - def test_lstm_sequence(self): - """Test LSTMCell over a sequence of inputs.""" - module = nnx.LSTMCell( - in_features=3, - hidden_features=4, - rngs=nnx.Rngs(0), - ) - x = random.normal(random.PRNGKey(1), (5, 2, 3)) # seq_len, batch, feature - carry = module.initialize_carry(x.shape[1:], module.rngs) - outputs = [] - for t in range(x.shape[0]): - carry, y = module(carry, x[t]) - outputs.append(y) - outputs = jnp.stack(outputs) - self.assertEqual(outputs.shape, (5, 2, 4)) - - def test_lstm_with_different_dtypes(self): - """Test LSTMCell with different data types.""" - module = nnx.LSTMCell( - in_features=3, - hidden_features=4, - dtype=jnp.bfloat16, - param_dtype=jnp.bfloat16, - rngs=nnx.Rngs(0), - ) - x = jnp.ones((2, 3), dtype=jnp.bfloat16) - carry = module.initialize_carry(x.shape, module.rngs) - new_carry, y = module(carry, x) - self.assertEqual(y.dtype, jnp.bfloat16) - self.assertEqual(y.shape, (2, 4)) - - def test_lstm_with_custom_activations(self): - """Test LSTMCell with custom activation functions.""" - module = nnx.LSTMCell( - in_features=3, - hidden_features=4, - gate_fn=jax.nn.relu, - activation_fn=jax.nn.elu, - rngs=nnx.Rngs(0), - ) - x = jnp.ones((1, 3)) - carry = module.initialize_carry(x.shape, module.rngs) - new_carry, y = module(carry, x) - self.assertEqual(y.shape, (1, 4)) - - def test_lstm_initialize_carry(self): - """Test the initialize_carry method.""" - module = nnx.LSTMCell( - in_features=3, - hidden_features=4, - carry_init=initializers.ones, - rngs=nnx.Rngs(0), - ) - x_shape = (1, 3) - carry = module.initialize_carry(x_shape, module.rngs) - c, h = carry - self.assertTrue(jnp.all(c == 1.0)) - self.assertTrue(jnp.all(h == 1.0)) - self.assertEqual(c.shape, (1, 4)) - self.assertEqual(h.shape, (1, 4)) - - def test_lstm_with_variable_sequence_length(self): - """Test LSTMCell with variable sequence lengths.""" - module = nnx.LSTMCell(in_features=3, hidden_features=4, rngs=nnx.Rngs(0)) - - # Simulate a batch with variable sequence lengths - x = jnp.array( - [ - [[1, 2, 3], [4, 5, 6], [0, 0, 0]], # Sequence length 2 - [[7, 8, 9], [10, 11, 12], [13, 14, 15]], # Sequence length 3 - ] - ) # Shape: (batch_size=2, max_seq_length=3, features=3) - - seq_lengths = jnp.array([2, 3]) # Actual lengths for each sequence - batch_size = x.shape[0] - max_seq_length = x.shape[1] - carry = module.initialize_carry((batch_size, 3), module.rngs) - outputs = [] - for t in range(max_seq_length): - input_t = x[:, t, :] - carry, y = module(carry, input_t) - outputs.append(y) - outputs = jnp.stack( - outputs, axis=1 - ) # Shape: (batch_size, max_seq_length, hidden_features) - - # Zero out outputs beyond the actual sequence lengths - mask = jnp.arange(max_seq_length)[None, :] < seq_lengths[:, None] - outputs = outputs * mask[:, :, None] - self.assertEqual(outputs.shape, (2, 3, 4)) - - def test_lstm_stateful(self): - """Test that LSTMCell maintains state across calls.""" - module = nnx.LSTMCell( - in_features=3, - hidden_features=4, - rngs=nnx.Rngs(0), - ) - x1 = jnp.ones((1, 3)) - x2 = jnp.ones((1, 3)) * 2 - carry = module.initialize_carry(x1.shape) - carry, y1 = module(carry, x1) - carry, y2 = module(carry, x2) - self.assertEqual(y1.shape, (1, 4)) - self.assertEqual(y2.shape, (1, 4)) - - def test_lstm_equivalence_with_flax_linen(self): - """Test that nnx.LSTMCell produces the same outputs as flax.linen.LSTMCell.""" - in_features = 3 - hidden_features = 4 - key = random.PRNGKey(42) - x = random.normal(key, (1, in_features)) - - # Initialize nnx.LSTMCell - rngs_nnx = nnx.Rngs(0) - module_nnx = nnx.LSTMCell( - in_features=in_features, - hidden_features=hidden_features, - rngs=rngs_nnx, - ) - carry_nnx = module_nnx.initialize_carry(x.shape, rngs_nnx) - # Initialize flax.linen.LSTMCell - module_linen = linen.LSTMCell( - features=hidden_features, - ) - carry_linen = module_linen.initialize_carry(random.PRNGKey(0), x.shape) - variables_linen = module_linen.init(random.PRNGKey(1), carry_linen, x) - - # Copy parameters from flax.linen.LSTMCell to nnx.LSTMCell - params_linen = variables_linen['params'] - # Map the parameters from linen to nnx - # Assuming the parameter names and shapes are compatible - # For a precise mapping, you might need to adjust parameter names - # Get the parameters from nnx module - nnx_params = module_nnx.__dict__ - - # Map parameters from linen to nnx - for gate in ['i', 'f', 'g', 'o']: - # Input kernels (input to gate) - if gate == 'f': - nnx_layer = getattr(module_nnx, f'if_') - else: - nnx_layer = getattr(module_nnx, f'i{gate}') - linen_params = params_linen[f'i{gate}'] - nnx_layer.kernel.value = linen_params['kernel'] - if nnx_layer.use_bias: - nnx_layer.bias.value = linen_params['bias'] - # Hidden kernels (hidden state to gate) - nnx_layer = getattr(module_nnx, f'h{gate}') - linen_params = params_linen[f'h{gate}'] - nnx_layer.kernel.value = linen_params['kernel'] - if nnx_layer.use_bias: - nnx_layer.bias.value = linen_params['bias'] - - # Run both modules - new_carry_nnx, y_nnx = module_nnx(carry_nnx, x) - new_carry_linen, y_linen = module_linen.apply( - variables_linen, carry_linen, x - ) - - # Compare outputs - np.testing.assert_allclose(y_nnx, y_linen, atol=1e-5) - # Compare carries - for c_nnx, c_linen in zip(new_carry_nnx, new_carry_linen): - np.testing.assert_allclose(c_nnx, c_linen, atol=1e-5) + def test_basic(self): + module = nnx.LSTMCell( + in_features=3, + hidden_features=4, + rngs=nnx.Rngs(0), + ) + x = jnp.ones((2, 3)) + carry = module.initialize_carry(x.shape, module.rngs) + new_carry, y = module(carry, x) + self.assertEqual(y.shape, (2, 4)) + + def test_lstm_sequence(self): + """Test LSTMCell over a sequence of inputs.""" + module = nnx.LSTMCell( + in_features=3, + hidden_features=4, + rngs=nnx.Rngs(0), + ) + x = random.normal(random.PRNGKey(1), (5, 2, 3)) # seq_len, batch, feature + carry = module.initialize_carry(x.shape[1:], module.rngs) + outputs = [] + for t in range(x.shape[0]): + carry, y = module(carry, x[t]) + outputs.append(y) + outputs = jnp.stack(outputs) + self.assertEqual(outputs.shape, (5, 2, 4)) + + def test_lstm_with_different_dtypes(self): + """Test LSTMCell with different data types.""" + module = nnx.LSTMCell( + in_features=3, + hidden_features=4, + dtype=jnp.bfloat16, + param_dtype=jnp.bfloat16, + rngs=nnx.Rngs(0), + ) + x = jnp.ones((2, 3), dtype=jnp.bfloat16) + carry = module.initialize_carry(x.shape, module.rngs) + new_carry, y = module(carry, x) + self.assertEqual(y.dtype, jnp.bfloat16) + self.assertEqual(y.shape, (2, 4)) + + def test_lstm_with_custom_activations(self): + """Test LSTMCell with custom activation functions.""" + module = nnx.LSTMCell( + in_features=3, + hidden_features=4, + gate_fn=jax.nn.relu, + activation_fn=jax.nn.elu, + rngs=nnx.Rngs(0), + ) + x = jnp.ones((1, 3)) + carry = module.initialize_carry(x.shape, module.rngs) + new_carry, y = module(carry, x) + self.assertEqual(y.shape, (1, 4)) + + def test_lstm_initialize_carry(self): + """Test the initialize_carry method.""" + module = nnx.LSTMCell( + in_features=3, + hidden_features=4, + carry_init=initializers.ones, + rngs=nnx.Rngs(0), + ) + x_shape = (1, 3) + carry = module.initialize_carry(x_shape, module.rngs) + c, h = carry + self.assertTrue(jnp.all(c == 1.0)) + self.assertTrue(jnp.all(h == 1.0)) + self.assertEqual(c.shape, (1, 4)) + self.assertEqual(h.shape, (1, 4)) + + def test_lstm_with_variable_sequence_length(self): + """Test LSTMCell with variable sequence lengths.""" + module = nnx.LSTMCell( + in_features=3, + hidden_features=4, + rngs=nnx.Rngs(0) + ) + # Simulate a batch with variable sequence lengths + x = jnp.array([ + [[1, 2, 3], [4, 5, 6], [0, 0, 0]], # Sequence length 2 + [[7, 8, 9], [10, 11, 12], [13, 14, 15]], # Sequence length 3 + ]) # Shape: (batch_size=2, max_seq_length=3, features=3) + + seq_lengths = jnp.array([2, 3]) # Actual lengths for each sequence + batch_size = x.shape[0] + max_seq_length = x.shape[1] + carry = module.initialize_carry((batch_size, 3), module.rngs) + outputs = [] + for t in range(max_seq_length): + input_t = x[:, t, :] + carry, y = module(carry, input_t) + outputs.append(y) + outputs = jnp.stack(outputs, axis=1) # Shape: (batch_size, max_seq_length, hidden_features) + + # Zero out outputs beyond the actual sequence lengths + mask = (jnp.arange(max_seq_length)[None, :] < seq_lengths[:, None]) + outputs = outputs * mask[:, :, None] + self.assertEqual(outputs.shape, (2, 3, 4)) + + def test_lstm_stateful(self): + """Test that LSTMCell maintains state across calls.""" + module = nnx.LSTMCell( + in_features=3, + hidden_features=4, + rngs=nnx.Rngs(0), + ) + x1 = jnp.ones((1, 3)) + x2 = jnp.ones((1, 3)) * 2 + carry = module.initialize_carry(x1.shape) + carry, y1 = module(carry, x1) + carry, y2 = module(carry, x2) + self.assertEqual(y1.shape, (1, 4)) + self.assertEqual(y2.shape, (1, 4)) + + def test_lstm_equivalence_with_flax_linen(self): + """Test that nnx.LSTMCell produces the same outputs as flax.linen.LSTMCell.""" + in_features = 3 + hidden_features = 4 + key = random.PRNGKey(42) + x = random.normal(key, (1, in_features)) + + # Initialize nnx.LSTMCell + rngs_nnx = nnx.Rngs(0) + module_nnx = nnx.LSTMCell( + in_features=in_features, + hidden_features=hidden_features, + rngs=rngs_nnx, + ) + carry_nnx = module_nnx.initialize_carry(x.shape, rngs_nnx) + # Initialize flax.linen.LSTMCell + module_linen = linen.LSTMCell( + features=hidden_features, + ) + carry_linen = module_linen.initialize_carry(random.PRNGKey(0), x.shape) + variables_linen = module_linen.init(random.PRNGKey(1), carry_linen, x) + + # Copy parameters from flax.linen.LSTMCell to nnx.LSTMCell + params_linen = variables_linen['params'] + # Map the parameters from linen to nnx + # Assuming the parameter names and shapes are compatible + # For a precise mapping, you might need to adjust parameter names + # Get the parameters from nnx module + nnx_params = module_nnx.__dict__ + + # Map parameters from linen to nnx + for gate in ['i', 'f', 'g', 'o']: + # Input kernels (input to gate) + if gate == 'f': + nnx_layer = getattr(module_nnx, f'if_') + else: + nnx_layer = getattr(module_nnx, f'i{gate}') + linen_params = params_linen[f'i{gate}'] + nnx_layer.kernel.value = linen_params['kernel'] + if nnx_layer.use_bias: + nnx_layer.bias.value = linen_params['bias'] + # Hidden kernels (hidden state to gate) + nnx_layer = getattr(module_nnx, f'h{gate}') + linen_params = params_linen[f'h{gate}'] + nnx_layer.kernel.value = linen_params['kernel'] + if nnx_layer.use_bias: + nnx_layer.bias.value = linen_params['bias'] + + # Run both modules + new_carry_nnx, y_nnx = module_nnx(carry_nnx, x) + new_carry_linen, y_linen = module_linen.apply(variables_linen, carry_linen, x) + + # Compare outputs + np.testing.assert_allclose(y_nnx, y_linen, atol=1e-5) + # Compare carries + for c_nnx, c_linen in zip(new_carry_nnx, new_carry_linen): + np.testing.assert_allclose(c_nnx, c_linen, atol=1e-5) class TestRNN(absltest.TestCase): - def test_rnn_with_lstm_cell(self): - """Test RNN module using LSTMCell.""" - # Initialize the LSTMCell - cell = nnx.LSTMCell( - in_features=3, - hidden_features=4, - rngs=nnx.Rngs(0), - ) - - # Initialize the RNN module with the LSTMCell - rnn = nnx.RNN(cell) - - # Create input data (batch_size=2, seq_length=5, features=3) - x = jnp.ones((2, 5, 3)) - - # Initialize the carry - carry = cell.initialize_carry((2, 3), cell.rngs) - - # Run the RNN module - outputs = rnn(x, initial_carry=carry) - - self.assertEqual( - outputs.shape, (2, 5, 4) - ) # Output features should match hidden_features - - def test_rnn_with_gru_cell(self): - """Test RNN module using GRUCell.""" - # Initialize the GRUCell - cell = nnx.GRUCell( - in_features=3, - hidden_features=4, - rngs=nnx.Rngs(1), - ) - - # Initialize the RNN module with the GRUCell - rnn = nnx.RNN(cell) - - # Create input data (batch_size=2, seq_length=5, features=3) - x = jnp.ones((2, 5, 3)) - - # Initialize the carry - carry = cell.initialize_carry((2, 3), cell.rngs) - - # Run the RNN module - outputs = rnn(x, initial_carry=carry) - - self.assertEqual( - outputs.shape, (2, 5, 4) - ) # Output features should match hidden_features - - def test_rnn_time_major(self): - """Test RNN module with time_major=True.""" - # Initialize the LSTMCell - cell = nnx.LSTMCell( - in_features=3, - hidden_features=4, - rngs=nnx.Rngs(2), - ) - - # Initialize the RNN module with time_major=True - rnn = nnx.RNN(cell, time_major=True) - - # Create input data (seq_length=5, batch_size=2, features=3) - x = jnp.ones((5, 2, 3)) - - # Initialize the carry - carry = cell.initialize_carry(x.shape[1:2] + x.shape[2:], cell.rngs) - - # Run the RNN module - outputs = rnn(x, initial_carry=carry) - - self.assertEqual( - outputs.shape, (5, 2, 4) - ) # Output features should match hidden_features - - def test_rnn_reverse(self): - """Test RNN module with reverse=True.""" - # Initialize the LSTMCell - cell = nnx.LSTMCell( - in_features=3, - hidden_features=4, - rngs=nnx.Rngs(3), - ) - - # Initialize the RNN module with reverse=True - rnn = nnx.RNN(cell, reverse=True) - - # Create input data (batch_size=2, seq_length=5, features=3) - x = jnp.tile(jnp.arange(5), (2, 1)).reshape( - 2, 5, 1 - ) # Distinct values to check reversal - x = jnp.concatenate([x, x, x], axis=-1) # Shape: (2, 5, 3) - - # Run the RNN module - outputs = rnn(x) - - # Check if the outputs are in reverse order - outputs_reversed = outputs[:, ::-1, :] - # Since we used distinct input values, we can compare outputs to check reversal - # For simplicity, just check the shapes here - self.assertEqual(outputs.shape, (2, 5, 4)) - self.assertEqual(outputs_reversed.shape, (2, 5, 4)) - - def test_rnn_with_seq_lengths(self): - """Test RNN module with variable sequence lengths.""" - # Initialize the LSTMCell - cell = nnx.LSTMCell( - in_features=3, - hidden_features=4, - rngs=nnx.Rngs(4), - ) - - # Initialize the RNN module - rnn = nnx.RNN(cell, return_carry=True) - - # Create input data with padding (batch_size=2, seq_length=5, features=3) - x = jnp.array( - [ - [ - [1, 1, 1], - [2, 2, 2], - [3, 3, 3], - [0, 0, 0], - [0, 0, 0], - ], # Sequence length 3 - [ - [4, 4, 4], - [5, 5, 5], - [6, 6, 6], - [7, 7, 7], - [8, 8, 8], - ], # Sequence length 5 - ] - ) # Shape: (2, 5, 3) - - seq_lengths = jnp.array([3, 5]) # Actual lengths for each sequence - - # Initialize the carry - carry = cell.initialize_carry((2, 3), cell.rngs) - - # Run the RNN module - final_carry, outputs = rnn(x, initial_carry=carry, seq_lengths=seq_lengths) - - self.assertEqual(outputs.shape, (2, 5, 4)) - - self.assertEqual( - final_carry[0].shape, (2, 4) - ) # c: (batch_size, hidden_features) - self.assertEqual( - final_carry[1].shape, (2, 4) - ) # h: (batch_size, hidden_features) - - # Todo: a better test by matching the outputs with the expected values - - def test_rnn_with_keep_order(self): - """Test RNN module with reverse=True and keep_order=True.""" - # Initialize the LSTMCell - cell = nnx.LSTMCell( - in_features=3, - hidden_features=4, - rngs=nnx.Rngs(5), - ) - - # Initialize the RNN module with reverse=True and keep_order=True - rnn = nnx.RNN(cell, reverse=True, keep_order=True) - - # Create input data (batch_size=2, seq_length=5, features=3) - x = jnp.tile(jnp.arange(5), (2, 1)).reshape( - 2, 5, 1 - ) # Distinct values to check reversal - x = jnp.concatenate([x, x, x], axis=-1) # Shape: (2, 5, 3) - - # Initialize the carry - carry = cell.initialize_carry((2, 3), cell.rngs) - - # Run the RNN module - outputs = rnn(x, initial_carry=carry) - - # Check if the outputs are in the original order despite processing in reverse - self.assertEqual(outputs.shape, (2, 5, 4)) - - def test_rnn_equivalence_with_flax_linen(self): - """Test that nnx.RNN produces the same outputs as flax.linen.RNN.""" - in_features = 3 - hidden_features = 4 - seq_length = 5 - batch_size = 2 - key = random.PRNGKey(42) - - # Create input data - x = random.normal(key, (batch_size, seq_length, in_features)) - - # Initialize nnx.LSTMCell and RNN - rngs_nnx = nnx.Rngs(0) - cell_nnx = nnx.LSTMCell( - in_features=in_features, - hidden_features=hidden_features, - rngs=rngs_nnx, - ) - rnn_nnx = nnx.RNN(cell_nnx) - - # Initialize flax.linen.LSTMCell and RNN - cell_linen = linen.LSTMCell(features=hidden_features) - rnn_linen = linen.RNN(cell_linen) - carry_linen = cell_linen.initialize_carry(random.PRNGKey(0), x[:, 0].shape) - variables_linen = rnn_linen.init(random.PRNGKey(1), x) - - # Copy parameters from flax.linen to nnx - params_linen = variables_linen['params']['cell'] - # Copy cell parameters - for gate in ['i', 'f', 'g', 'o']: - # Input kernels - if gate == 'f': - nnx_layer = getattr(cell_nnx, f'if_') - else: - nnx_layer = getattr(cell_nnx, f'i{gate}') - linen_params = params_linen[f'i{gate}'] - nnx_layer.kernel.value = linen_params['kernel'] - if nnx_layer.use_bias: - nnx_layer.bias.value = linen_params['bias'] - # Hidden kernels - nnx_layer = getattr(cell_nnx, f'h{gate}') - linen_params = params_linen[f'h{gate}'] - nnx_layer.kernel.value = linen_params['kernel'] - if nnx_layer.use_bias: - nnx_layer.bias.value = linen_params['bias'] - - # Initialize carries - carry_nnx = cell_nnx.initialize_carry((batch_size, in_features), rngs_nnx) - - # Run nnx.RNN - outputs_nnx = rnn_nnx(x, initial_carry=carry_nnx) - - # Run flax.linen.RNN - outputs_linen = rnn_linen.apply( - variables_linen, x, initial_carry=carry_linen - ) - - # Compare outputs - np.testing.assert_allclose(outputs_nnx, outputs_linen, atol=1e-5) - - def test_rnn_with_unroll(self): - """Test RNN module with unroll parameter.""" - # Initialize the LSTMCell - cell = nnx.LSTMCell(in_features=3, hidden_features=4, rngs=nnx.Rngs(6)) - - # Initialize the RNN module with unroll=2 - rnn = nnx.RNN(cell, unroll=2) - - # Create input data (batch_size=2, seq_length=6, features=3) - x = jnp.ones((2, 6, 3)) - - # Initialize the carry - carry = cell.initialize_carry((2, 3), cell.rngs) - - # Run the RNN module - outputs = rnn(x, initial_carry=carry) - - self.assertEqual( - outputs.shape, (2, 6, 4) - ) # Output features should match hidden_features - - def test_rnn_with_custom_cell(self): - """Test RNN module with a custom RNN cell.""" - - class CustomRNNCell(nnx.Module): - """A simple custom RNN cell.""" - - in_features: int - hidden_features: int - - def __init__(self, in_features, hidden_features, rngs): - self.in_features = in_features - self.hidden_features = hidden_features - self.rngs = rngs - self.dense = nnx.Linear( - in_features=in_features + hidden_features, - out_features=hidden_features, - rngs=rngs, + + def test_rnn_with_lstm_cell(self): + """Test RNN module using LSTMCell.""" + # Initialize the LSTMCell + cell = nnx.LSTMCell( + in_features=3, + hidden_features=4, + rngs=nnx.Rngs(0), + ) + + # Initialize the RNN module with the LSTMCell + rnn = nnx.RNN(cell) + + # Create input data (batch_size=2, seq_length=5, features=3) + x = jnp.ones((2, 5, 3)) + + # Initialize the carry + carry = cell.initialize_carry((2, 3), cell.rngs) + + # Run the RNN module + outputs = rnn(x, initial_carry=carry) + + self.assertEqual(outputs.shape, (2, 5, 4)) # Output features should match hidden_features + + def test_rnn_with_gru_cell(self): + """Test RNN module using GRUCell.""" + # Initialize the GRUCell + cell = nnx.GRUCell( + in_features=3, + hidden_features=4, + rngs=nnx.Rngs(1), + ) + + # Initialize the RNN module with the GRUCell + rnn = nnx.RNN(cell) + + # Create input data (batch_size=2, seq_length=5, features=3) + x = jnp.ones((2, 5, 3)) + + # Initialize the carry + carry = cell.initialize_carry((2, 3), cell.rngs) + + # Run the RNN module + outputs = rnn(x, initial_carry=carry) + + self.assertEqual(outputs.shape, (2, 5, 4)) # Output features should match hidden_features + + def test_rnn_time_major(self): + """Test RNN module with time_major=True.""" + # Initialize the LSTMCell + cell = nnx.LSTMCell( + in_features=3, + hidden_features=4, + rngs=nnx.Rngs(2), + ) + + # Initialize the RNN module with time_major=True + rnn = nnx.RNN(cell, time_major=True) + + # Create input data (seq_length=5, batch_size=2, features=3) + x = jnp.ones((5, 2, 3)) + + # Initialize the carry + carry = cell.initialize_carry(x.shape[1:2] + x.shape[2:], cell.rngs) + + # Run the RNN module + outputs = rnn(x, initial_carry=carry) + + self.assertEqual(outputs.shape, (5, 2, 4)) # Output features should match hidden_features + + def test_rnn_reverse(self): + """Test RNN module with reverse=True.""" + # Initialize the LSTMCell + cell = nnx.LSTMCell( + in_features=3, + hidden_features=4, + rngs=nnx.Rngs(3), ) - def __call__(self, carry, inputs): - h = carry - x = jnp.concatenate([inputs, h], axis=-1) - new_h = jax.nn.tanh(self.dense(x)) - return new_h, new_h - - def initialize_carry(self, input_shape, rngs): - batch_size = input_shape[0] - h = jnp.zeros((batch_size, self.hidden_features)) - return h - - @property - def num_feature_axes(self) -> int: - return 1 - - # Initialize the custom RNN cell - cell = CustomRNNCell(in_features=3, hidden_features=4, rngs=nnx.Rngs(7)) - - # Initialize the RNN module - rnn = nnx.RNN(cell) - - # Create input data (batch_size=2, seq_length=5, features=3) - x = jnp.ones((2, 5, 3)) - - # Initialize the carry - carry = cell.initialize_carry((2, 3), cell.rngs) - - # Run the RNN module - outputs = rnn(x, initial_carry=carry) - - self.assertEqual( - outputs.shape, (2, 5, 4) - ) # Output features should match hidden_features - - def test_rnn_with_different_dtypes(self): - """Test RNN module with different data types.""" - # Initialize the LSTMCell with float16 - cell = nnx.LSTMCell( - in_features=3, - hidden_features=4, - dtype=jnp.float16, - param_dtype=jnp.float16, - rngs=nnx.Rngs(8), - ) - - # Initialize the RNN module - rnn = nnx.RNN(cell) - - # Create input data (batch_size=2, seq_length=5, features=3) - x = jnp.ones((2, 5, 3), dtype=jnp.float16) - - # Initialize the carry - carry = cell.initialize_carry((2, 3), cell.rngs) - - # Run the RNN module - outputs = rnn(x, initial_carry=carry) - - self.assertEqual(outputs.dtype, jnp.float16) - self.assertEqual(outputs.shape, (2, 5, 4)) - - def test_rnn_with_variable_batch_size(self): - """Test RNN module with variable batch sizes.""" - # Initialize the LSTMCell - cell = nnx.LSTMCell( - in_features=3, - hidden_features=4, - rngs=nnx.Rngs(9), - ) - - # Initialize the RNN module - rnn = nnx.RNN(cell) - - for batch_size in [1, 2, 5]: - # Create input data (batch_size, seq_length=5, features=3) - x = jnp.ones((batch_size, 5, 3)) - - # Initialize the carry - carry = cell.initialize_carry((batch_size, 3), cell.rngs) - - # Run the RNN module - outputs = rnn(x, initial_carry=carry) - - self.assertEqual(outputs.shape, (batch_size, 5, 4)) - - def test_recurrent_dropout(self): - class LSTMWithRecurrentDropout(nnx.OptimizedLSTMCell): - def __init__( - self, - *, - rngs: nnx.Rngs, - in_features: int, - hidden_features: int, - dropout_rate: float, - **kwargs, - ): - super().__init__( - in_features=in_features, - hidden_features=hidden_features, - rngs=rngs, - **kwargs, + # Initialize the RNN module with reverse=True + rnn = nnx.RNN(cell, reverse=True) + + # Create input data (batch_size=2, seq_length=5, features=3) + x = jnp.tile(jnp.arange(5), (2, 1)).reshape(2, 5, 1) # Distinct values to check reversal + x = jnp.concatenate([x, x, x], axis=-1) # Shape: (2, 5, 3) + + # Run the RNN module + outputs = rnn(x) + + # Check if the outputs are in reverse order + outputs_reversed = outputs[:, ::-1, :] + # Since we used distinct input values, we can compare outputs to check reversal + # For simplicity, just check the shapes here + self.assertEqual(outputs.shape, (2, 5, 4)) + self.assertEqual(outputs_reversed.shape, (2, 5, 4)) + + def test_rnn_with_seq_lengths(self): + """Test RNN module with variable sequence lengths.""" + # Initialize the LSTMCell + cell = nnx.LSTMCell( + in_features=3, + hidden_features=4, + rngs=nnx.Rngs(4), ) - self.recurrent_dropout = nnx.Dropout( - rate=dropout_rate, rng_collection='recurrent_dropout', rngs=rngs + + # Initialize the RNN module + rnn = nnx.RNN(cell, return_carry=True) + + # Create input data with padding (batch_size=2, seq_length=5, features=3) + x = jnp.array([ + [[1, 1, 1], [2, 2, 2], [3, 3, 3], [0, 0, 0], [0, 0, 0]], # Sequence length 3 + [[4, 4, 4], [5, 5, 5], [6, 6, 6], [7, 7, 7], [8, 8, 8]], # Sequence length 5 + ]) # Shape: (2, 5, 3) + + seq_lengths = jnp.array([3, 5]) # Actual lengths for each sequence + + # Initialize the carry + carry = cell.initialize_carry((2, 3), cell.rngs) + + # Run the RNN module + final_carry, outputs = rnn(x, initial_carry=carry, seq_lengths=seq_lengths) + + self.assertEqual(outputs.shape, (2, 5, 4)) + + self.assertEqual(final_carry[0].shape, (2, 4)) # c: (batch_size, hidden_features) + self.assertEqual(final_carry[1].shape, (2, 4)) # h: (batch_size, hidden_features) + + # Todo: a better test by matching the outputs with the expected values + + def test_rnn_with_keep_order(self): + """Test RNN module with reverse=True and keep_order=True.""" + # Initialize the LSTMCell + cell = nnx.LSTMCell( + in_features=3, + hidden_features=4, + rngs=nnx.Rngs(5), + ) + + # Initialize the RNN module with reverse=True and keep_order=True + rnn = nnx.RNN(cell, reverse=True, keep_order=True) + + # Create input data (batch_size=2, seq_length=5, features=3) + x = jnp.tile(jnp.arange(5), (2, 1)).reshape(2, 5, 1) # Distinct values to check reversal + x = jnp.concatenate([x, x, x], axis=-1) # Shape: (2, 5, 3) + + # Initialize the carry + carry = cell.initialize_carry((2, 3), cell.rngs) + + # Run the RNN module + outputs = rnn(x, initial_carry=carry) + + # Check if the outputs are in the original order despite processing in reverse + self.assertEqual(outputs.shape, (2, 5, 4)) + + def test_rnn_equivalence_with_flax_linen(self): + """Test that nnx.RNN produces the same outputs as flax.linen.RNN.""" + in_features = 3 + hidden_features = 4 + seq_length = 5 + batch_size = 2 + key = random.PRNGKey(42) + + # Create input data + x = random.normal(key, (batch_size, seq_length, in_features)) + + # Initialize nnx.LSTMCell and RNN + rngs_nnx = nnx.Rngs(0) + cell_nnx = nnx.LSTMCell( + in_features=in_features, + hidden_features=hidden_features, + rngs=rngs_nnx, + ) + rnn_nnx = nnx.RNN(cell_nnx) + + # Initialize flax.linen.LSTMCell and RNN + cell_linen = linen.LSTMCell(features=hidden_features) + rnn_linen = linen.RNN(cell_linen) + carry_linen = cell_linen.initialize_carry(random.PRNGKey(0), x[:, 0].shape) + variables_linen = rnn_linen.init(random.PRNGKey(1), x) + + # Copy parameters from flax.linen to nnx + params_linen = variables_linen['params']['cell'] + # Copy cell parameters + for gate in ['i', 'f', 'g', 'o']: + # Input kernels + if gate == 'f': + nnx_layer = getattr(cell_nnx, f'if_') + else: + nnx_layer = getattr(cell_nnx, f'i{gate}') + linen_params = params_linen[f'i{gate}'] + nnx_layer.kernel.value = linen_params['kernel'] + if nnx_layer.use_bias: + nnx_layer.bias.value = linen_params['bias'] + # Hidden kernels + nnx_layer = getattr(cell_nnx, f'h{gate}') + linen_params = params_linen[f'h{gate}'] + nnx_layer.kernel.value = linen_params['kernel'] + if nnx_layer.use_bias: + nnx_layer.bias.value = linen_params['bias'] + + # Initialize carries + carry_nnx = cell_nnx.initialize_carry((batch_size, in_features), rngs_nnx) + + # Run nnx.RNN + outputs_nnx = rnn_nnx(x, initial_carry=carry_nnx) + + # Run flax.linen.RNN + outputs_linen = rnn_linen.apply(variables_linen, x, initial_carry=carry_linen) + + # Compare outputs + np.testing.assert_allclose(outputs_nnx, outputs_linen, atol=1e-5) + + def test_rnn_with_unroll(self): + """Test RNN module with unroll parameter.""" + # Initialize the LSTMCell + cell = nnx.LSTMCell( + in_features=3, + hidden_features=4, + rngs=nnx.Rngs(6) + ) + + # Initialize the RNN module with unroll=2 + rnn = nnx.RNN(cell, unroll=2) + + # Create input data (batch_size=2, seq_length=6, features=3) + x = jnp.ones((2, 6, 3)) + + # Initialize the carry + carry = cell.initialize_carry((2, 3), cell.rngs) + + # Run the RNN module + outputs = rnn(x, initial_carry=carry) + + self.assertEqual(outputs.shape, (2, 6, 4)) # Output features should match hidden_features + + def test_rnn_with_custom_cell(self): + """Test RNN module with a custom RNN cell.""" + class CustomRNNCell(nnx.Module): + """A simple custom RNN cell.""" + + in_features: int + hidden_features: int + + def __init__(self, in_features, hidden_features, rngs): + self.in_features = in_features + self.hidden_features = hidden_features + self.rngs = rngs + self.dense = nnx.Linear( + in_features=in_features + hidden_features, + out_features=hidden_features, + rngs=rngs, + ) + + def __call__(self, carry, inputs): + h = carry + x = jnp.concatenate([inputs, h], axis=-1) + new_h = jax.nn.tanh(self.dense(x)) + return new_h, new_h + + def initialize_carry(self, input_shape, rngs): + batch_size = input_shape[0] + h = jnp.zeros((batch_size, self.hidden_features)) + return h + + @property + def num_feature_axes(self) -> int: + return 1 + + # Initialize the custom RNN cell + cell = CustomRNNCell( + in_features=3, + hidden_features=4, + rngs=nnx.Rngs(7) ) - def __call__(self, carry, x): - h, c = carry - new_h, new_c = super().__call__((h, c), x) - new_h = jax.tree.map(self.recurrent_dropout, new_h) - return new_h, new_c - - class RNNWithRecurrentDropout(nnx.Module): - def __init__( - self, - *, - rngs: nnx.Rngs, - in_features: int, - hidden_features: int = 32, - dropout_rate: float = 0.5, - recurrent_dropout_rate: float = 0.25, - ): - cell = LSTMWithRecurrentDropout( - in_features=in_features, - hidden_features=hidden_features, - rngs=rngs, - dropout_rate=recurrent_dropout_rate, + # Initialize the RNN module + rnn = nnx.RNN(cell) + + # Create input data (batch_size=2, seq_length=5, features=3) + x = jnp.ones((2, 5, 3)) + + # Initialize the carry + carry = cell.initialize_carry((2, 3), cell.rngs) + + # Run the RNN module + outputs = rnn(x, initial_carry=carry) + + self.assertEqual(outputs.shape, (2, 5, 4)) # Output features should match hidden_features + + def test_rnn_with_different_dtypes(self): + """Test RNN module with different data types.""" + # Initialize the LSTMCell with float16 + cell = nnx.LSTMCell( + in_features=3, + hidden_features=4, + dtype=jnp.float16, + param_dtype=jnp.float16, + rngs=nnx.Rngs(8), ) - self.lstm = nnx.RNN(cell, broadcast_rngs='recurrent_dropout') - self.dropout = nnx.Dropout(dropout_rate, rngs=rngs) - self.dense = nnx.Linear( - in_features=hidden_features, out_features=1, rngs=rngs + + # Initialize the RNN module + rnn = nnx.RNN(cell) + + # Create input data (batch_size=2, seq_length=5, features=3) + x = jnp.ones((2, 5, 3), dtype=jnp.float16) + + # Initialize the carry + carry = cell.initialize_carry((2, 3), cell.rngs) + + # Run the RNN module + outputs = rnn(x, initial_carry=carry) + + self.assertEqual(outputs.dtype, jnp.float16) + self.assertEqual(outputs.shape, (2, 5, 4)) + + def test_rnn_with_variable_batch_size(self): + """Test RNN module with variable batch sizes.""" + # Initialize the LSTMCell + cell = nnx.LSTMCell( + in_features=3, + hidden_features=4, + rngs=nnx.Rngs(9), ) - def __call__(self, x): - x = self.lstm(x) - x = self.dropout(x) - x = x[:, -1, :] # Use only the final hidden state - return self.dense(x) + # Initialize the RNN module + rnn = nnx.RNN(cell) - model = RNNWithRecurrentDropout( - in_features=32, - hidden_features=64, - dropout_rate=0.2, - recurrent_dropout_rate=0.1, - rngs=nnx.Rngs(0, recurrent_dropout=1), - ) + for batch_size in [1, 2, 5]: + # Create input data (batch_size, seq_length=5, features=3) + x = jnp.ones((batch_size, 5, 3)) - x = jnp.ones((8, 10, 32)) - self.assertEqual(model.lstm.cell.rngs.recurrent_dropout.count.value, 0) - y = model(x) + # Initialize the carry + carry = cell.initialize_carry((batch_size, 3), cell.rngs) - self.assertEqual(y.shape, (8, 1)) - self.assertEqual(model.lstm.cell.rngs.recurrent_dropout.count.value, 1) + # Run the RNN module + outputs = rnn(x, initial_carry=carry) + self.assertEqual(outputs.shape, (batch_size, 5, 4)) if __name__ == '__main__': - absltest.main() + absltest.main() diff --git a/tests/nnx/transforms_test.py b/tests/nnx/transforms_test.py index bfa461be39..10653ef20a 100644 --- a/tests/nnx/transforms_test.py +++ b/tests/nnx/transforms_test.py @@ -67,6 +67,27 @@ def g(m: Dict): assert m.a == 2 assert out == 1.0 + def test_simple_double_call(self): + n = 0 + m = nnx.Linear(2, 3, rngs=nnx.Rngs(0)) + + @nnx.jit + def f(m: nnx.Linear, x: jnp.ndarray) -> jnp.ndarray: + nonlocal n + n += 1 + return m(x) + + x = jnp.ones((1, 2)) + y = f(m, x) + + self.assertEqual(n, 1) + self.assertEqual(y.shape, (1, 3)) + + y = f(m, x) + + self.assertEqual(n, 1) + self.assertEqual(y.shape, (1, 3)) + def test_jit_on_init(self): n = 0 @@ -634,6 +655,9 @@ class Foo(nnx.Module): y: nnx.Param[jax.Array] z: int + def __hash__(self): + return id(self) + @nnx.custom_vjp def f(m: Foo): m.z += 1 @@ -674,6 +698,9 @@ class Foo(nnx.Module): y: nnx.Param[jax.Array] z: int + def __hash__(self): + return id(self) + x_in_path = nnx.PathContains('x') diff_state = nnx.DiffState(0, x_in_path) @@ -715,6 +742,9 @@ class Foo(nnx.Module): y: nnx.Param[jax.Array] z: int + def __hash__(self): + return id(self) + @nnx.custom_vjp @nnx.remat def f(m: Foo): @@ -760,6 +790,9 @@ class Foo(nnx.Module): y: nnx.Param[jax.Array] z: int + def __hash__(self): + return id(self) + @nnx.custom_vjp def f(m1: Foo, m2: Foo): m1.z += 1 @@ -813,6 +846,9 @@ class Foo(nnx.Module): y: nnx.Param[jax.Array] z: int + def __hash__(self): + return id(self) + @nnx.custom_vjp(nondiff_argnums=(0, 2)) def f(a, m: Foo, b): self.assertEqual(a, 1) @@ -1006,6 +1042,9 @@ def test_all_carry(self): class Foo(nnx.Module): n: nnx.BatchStat[int] + def __hash__(self): + return id(self) + foo = Foo(n=nnx.BatchStat(0)) @nnx.scan(in_axes=nnx.Carry, out_axes=nnx.Carry, length=3) @@ -1036,9 +1075,9 @@ def loop(foo: Foo, x): loop(foo, 0) def test_all_carry_new_reference_error(self): - @dataclasses.dataclass(repr=False) class Foo(nnx.Module): - n: nnx.BatchStat[int] + def __init__(self, n: nnx.BatchStat[int]): + self.n = n xs = jnp.arange(3) foo = Foo(n=nnx.BatchStat(0)) @@ -1056,9 +1095,9 @@ def loop(foo: Foo, x): loop(foo, xs) def test_all_scan(self): - @dataclasses.dataclass(repr=False) class Foo(nnx.Module): - n: nnx.BatchStat[jax.Array] + def __init__(self, n: nnx.BatchStat[jax.Array]): + self.n = n xs = jnp.arange(3) foo = Foo(n=nnx.BatchStat(jnp.arange(3))) @@ -1075,9 +1114,9 @@ def loop(foo: Foo, x): np.testing.assert_allclose(foo.n.value, jnp.arange(1, 4)) def test_all_broadcast(self): - @dataclasses.dataclass(repr=False) class Foo(nnx.Module): - n: nnx.BatchStat[int] + def __init__(self, n: nnx.BatchStat[int]): + self.n = n xs = jnp.array(1) foo = Foo(n=nnx.BatchStat(2)) @@ -1740,7 +1779,6 @@ def test_cache_tracing_object(self): x = jnp.arange(5) count = jnp.array(0) - @dataclasses.dataclass class Foo(nnx.Object): @nnx.split_rngs(splits=5) @@ -2696,6 +2734,9 @@ def zero(): class Foo(nnx.Object): timestep: TimeStep + def __hash__(self): + return id(self) + def update(self): def reward_2(self: Foo): self.timestep = TimeStep( @@ -2985,18 +3026,6 @@ def loop_fn(inputs): nnx.while_loop(lambda input: input[-1] > 0, while_loop_fn, (a, b, 2)) nnx.fori_loop(0, 2, fori_loop_fn, (a, b)) - def test_fori_output(self): - model = nnx.Linear(2, 2, rngs=nnx.Rngs(jax.random.PRNGKey(0))) - model2 = nnx.Linear(2, 2, rngs=nnx.Rngs(jax.random.PRNGKey(1))) - - def f(i, x): - return x - - model_out, model2_out = nnx.fori_loop(0, 10, f, (model, model2)) - - self.assertIs(model, model_out) - self.assertIs(model2, model2_out) - class TestSplitMergeInputs(absltest.TestCase): def test_split_inputs(self): @@ -3093,6 +3122,9 @@ def test_basic(self): class Foo(nnx.Module): a: nnx.Param + def __hash__(self): + return id(self) + @nnx.jit def f(m): y = jnp.sin(m.a.value) # error diff --git a/uv.lock b/uv.lock index 48bda4f756..bd7053e5ad 100644 --- a/uv.lock +++ b/uv.lock @@ -838,6 +838,12 @@ testing = [ { name = "treescope" }, ] +[package.dev-dependencies] +dev = [ + { name = "nanobind" }, + { name = "scikit-build-core" }, +] + [package.metadata] requires-dist = [ { name = "cloudpickle", marker = "extra == 'testing'", specifier = ">=3.0.0" }, @@ -890,11 +896,17 @@ requires-dist = [ { name = "tensorflow-text", marker = "platform_system != 'Darwin' and extra == 'testing'", specifier = ">=2.11.0" }, { name = "tensorstore" }, { name = "torch", marker = "extra == 'testing'" }, - { name = "treescope", specifier = ">=0.1.7" }, + { name = "treescope", specifier = ">=0.1.2" }, { name = "treescope", marker = "python_full_version >= '3.10' and extra == 'testing'", specifier = ">=0.1.1" }, { name = "typing-extensions", specifier = ">=4.2" }, ] +[package.metadata.requires-dev] +dev = [ + { name = "nanobind", specifier = ">=2.4.0" }, + { name = "scikit-build-core", extras = ["pyproject"], specifier = ">=0.10.7" }, +] + [[package]] name = "fonttools" version = "4.53.1" @@ -1935,6 +1947,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/59/7854fbfb59f8ae35483ce93493708be5942ebb6328cd85b3a609df629736/namex-0.0.8-py3-none-any.whl", hash = "sha256:7ddb6c2bb0e753a311b7590f84f6da659dd0c05e65cb89d519d54c0a250c0487", size = 5806 }, ] +[[package]] +name = "nanobind" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/01/a28722f6626e5c8a606dee71cb40c0b2ab9f7715b96bd34a9553c79dbf42/nanobind-2.4.0.tar.gz", hash = "sha256:a0392dee5f58881085b2ac8bfe8e53f74285aa4868b1472bfaf76cfb414e1c96", size = 953467 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/07/abff41fcade3613349eac71dacb166352babef515efd960a751e3175c262/nanobind-2.4.0-py3-none-any.whl", hash = "sha256:8cf27b04fbadeb9deb4a73f02bd838bf9f7e3e5a8ce44c50c93142b5728da58a", size = 232882 }, +] + [[package]] name = "nbclient" version = "0.10.0" @@ -2303,6 +2324,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c6/ac/dac4a63f978e4dcb3c6d3a78c4d8e0192a113d288502a1216950c41b1027/parso-0.8.4-py2.py3-none-any.whl", hash = "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18", size = 103650 }, ] +[[package]] +name = "pathspec" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191 }, +] + [[package]] name = "pexpect" version = "4.9.0" @@ -2956,6 +2986,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/ea/6f121d1802f3adae1981aea4209ea66f9d3c7f2f6d6b85ef4f13a61d17ef/rpds_py-0.20.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:bb273176be34a746bdac0b0d7e4e2c467323d13640b736c4c477881a3220a989", size = 213529 }, ] +[[package]] +name = "scikit-build-core" +version = "0.10.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/34/75/ad5664c8050bbbea46a5f2b6a3dfbc6e6cf284826c0eee0a12f861364b3f/scikit_build_core-0.10.7.tar.gz", hash = "sha256:04cbb59fe795202a7eeede1849112ee9dcbf3469feebd9b8b36aa541336ac4f8", size = 255019 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/fe/90476c4f6a1b2f922efa00d26e876dd40c7279e28ec18f08f0851ad21ba6/scikit_build_core-0.10.7-py3-none-any.whl", hash = "sha256:5e13ab7ca7c3c6dd019607c3a6f53cba67dade8757c4c4f75b459e2f90e4dbc3", size = 165511 }, +] + [[package]] name = "scikit-learn" version = "1.5.1" @@ -3669,14 +3714,14 @@ wheels = [ [[package]] name = "treescope" -version = "0.1.7" +version = "0.1.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/40/34/8ad5475c26837ca400c77951bcc0788b5f291d1509ae2eda5f97b042c24a/treescope-0.1.7.tar.gz", hash = "sha256:2c82ecb633f18d50e5809dd473703cf05aa074a4f3d1add74de7cf7ccdf81ae3", size = 530052 } +sdist = { url = "https://files.pythonhosted.org/packages/2f/5d/ecb176971c78d90a3f74b7878ab9d013995fed285e3386a503ca008c9b03/treescope-0.1.2.tar.gz", hash = "sha256:2e4b35780884dfdbdcf44315d1c1c98fcf41daa0ea48a5b45ecc716920f88c86", size = 402255 } wheels = [ - { url = "https://files.pythonhosted.org/packages/59/7d/f6da2b223749c58ec8ff95c87319196765fed05bd44dd86fb9bc4bf35f77/treescope-0.1.7-py3-none-any.whl", hash = "sha256:14e6527d4bfe6770ac9cbb8058e49b6685444d7cd0d3f85fd10c42491848b102", size = 175566 }, + { url = "https://files.pythonhosted.org/packages/af/11/1a4d1877e5f7202bb3d0778a77b6ca222848b9b36fa65cbbc1fe12cb82b7/treescope-0.1.2-py3-none-any.whl", hash = "sha256:1811df6fbf79a5f54804e3ce2230b100547dc6350c99d973a6b9ba2bcd932e57", size = 172154 }, ] [[package]]