{ "cells": [ { "cell_type": "markdown", "id": "d3014ad2", "metadata": {}, "source": [ "# Our Bloom for Sequence Classification using Prompt Tuning\n", "\n", "In this example, we showcase how Bloom model can be efficiently adapted in decentralized fashion. In particular, servers maintain the Bloom transformer, which is kept unchanged during adaptation, and learn only a few prefix tokens and a classification head. " ] }, { "cell_type": "markdown", "id": "243a8971", "metadata": {}, "source": [ "### Import all dependences and prepare the environment" ] }, { "cell_type": "code", "execution_count": 1, "id": "7e975fd7", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "env: CUDA_VISIBLE_DEVICES=6\n", "env: OMP_NUM_THREADS=1\n" ] } ], "source": [ "%env CUDA_VISIBLE_DEVICES=6\n", "%env OMP_NUM_THREADS=1\n", "%load_ext autoreload\n", "%autoreload 2\n", "\n", "import sys\n", "sys.path.append('..')\n", "\n", "# General \n", "import torch\n", "import pandas as pd\n", "from tqdm.auto import tqdm\n", "from torch.utils.data import DataLoader\n", "\n", "# Distributed\n", "from src.bloom.model import BloomForSequenceClassification\n", "\n", "# HF imports\n", "import transformers\n", "import datasets\n", "from datasets import load_dataset, load_metric\n", "\n", "# Visualization dependencies\n", "from IPython.display import clear_output\n", "import matplotlib.pyplot as plt\n", "%matplotlib inline\n", "\n", "plt.style.use('seaborn-whitegrid')\n", "plt.rcParams['pdf.fonttype'] = 42\n", "plt.rcParams['ps.fonttype'] = 42\n", "\n", "def print_params(model):\n", " for n, p in model.named_parameters():\n", " print(n, p.requires_grad, p.device)\n", " if p.requires_grad:\n", " print(p)" ] }, { "cell_type": "markdown", "id": "fa04b0b4", "metadata": {}, "source": [ "### Config" ] }, { "cell_type": "code", "execution_count": 2, "id": "291f6021", "metadata": {}, "outputs": [], "source": [ "MODEL_NAME='bigscience/bloom-6b3'\n", "\n", "PROMPT_TUNING_TYPE='deep'\n", "NUM_PREFIX_TOKENS=16\n", "NUM_LABELS=2\n", "DEVICE='cuda'\n", "BATCH_SIZE=32" ] }, { "cell_type": "markdown", "id": "006c934d", "metadata": {}, "source": [ "### Prepare the distributed Bloom model" ] }, { "cell_type": "code", "execution_count": 3, "id": "bff5d710", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Some weights of BloomForSequenceClassification were not initialized from the model checkpoint at bigscience/bloom-6b3 and are newly initialized: ['intermediate_prompt_embeddings.weight', 'score.weight', 'prompt_embeddings.weight']\n", "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "transformer.word_embeddings.weight False cuda:0\n", "transformer.word_embeddings_layernorm.weight False cuda:0\n", "transformer.word_embeddings_layernorm.bias False cuda:0\n", "transformer.h.0.input_layernorm.weight False cuda:0\n", "transformer.h.0.input_layernorm.bias False cuda:0\n", "transformer.h.0.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.0.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.0.self_attention.dense.weight False cuda:0\n", "transformer.h.0.self_attention.dense.bias False cuda:0\n", "transformer.h.0.post_attention_layernorm.weight False cuda:0\n", "transformer.h.0.post_attention_layernorm.bias False cuda:0\n", "transformer.h.0.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.0.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.0.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.0.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.1.input_layernorm.weight False cuda:0\n", "transformer.h.1.input_layernorm.bias False cuda:0\n", "transformer.h.1.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.1.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.1.self_attention.dense.weight False cuda:0\n", "transformer.h.1.self_attention.dense.bias False cuda:0\n", "transformer.h.1.post_attention_layernorm.weight False cuda:0\n", "transformer.h.1.post_attention_layernorm.bias False cuda:0\n", "transformer.h.1.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.1.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.1.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.1.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.2.input_layernorm.weight False cuda:0\n", "transformer.h.2.input_layernorm.bias False cuda:0\n", "transformer.h.2.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.2.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.2.self_attention.dense.weight False cuda:0\n", "transformer.h.2.self_attention.dense.bias False cuda:0\n", "transformer.h.2.post_attention_layernorm.weight False cuda:0\n", "transformer.h.2.post_attention_layernorm.bias False cuda:0\n", "transformer.h.2.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.2.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.2.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.2.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.3.input_layernorm.weight False cuda:0\n", "transformer.h.3.input_layernorm.bias False cuda:0\n", "transformer.h.3.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.3.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.3.self_attention.dense.weight False cuda:0\n", "transformer.h.3.self_attention.dense.bias False cuda:0\n", "transformer.h.3.post_attention_layernorm.weight False cuda:0\n", "transformer.h.3.post_attention_layernorm.bias False cuda:0\n", "transformer.h.3.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.3.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.3.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.3.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.4.input_layernorm.weight False cuda:0\n", "transformer.h.4.input_layernorm.bias False cuda:0\n", "transformer.h.4.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.4.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.4.self_attention.dense.weight False cuda:0\n", "transformer.h.4.self_attention.dense.bias False cuda:0\n", "transformer.h.4.post_attention_layernorm.weight False cuda:0\n", "transformer.h.4.post_attention_layernorm.bias False cuda:0\n", "transformer.h.4.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.4.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.4.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.4.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.5.input_layernorm.weight False cuda:0\n", "transformer.h.5.input_layernorm.bias False cuda:0\n", "transformer.h.5.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.5.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.5.self_attention.dense.weight False cuda:0\n", "transformer.h.5.self_attention.dense.bias False cuda:0\n", "transformer.h.5.post_attention_layernorm.weight False cuda:0\n", "transformer.h.5.post_attention_layernorm.bias False cuda:0\n", "transformer.h.5.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.5.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.5.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.5.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.6.input_layernorm.weight False cuda:0\n", "transformer.h.6.input_layernorm.bias False cuda:0\n", "transformer.h.6.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.6.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.6.self_attention.dense.weight False cuda:0\n", "transformer.h.6.self_attention.dense.bias False cuda:0\n", "transformer.h.6.post_attention_layernorm.weight False cuda:0\n", "transformer.h.6.post_attention_layernorm.bias False cuda:0\n", "transformer.h.6.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.6.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.6.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.6.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.7.input_layernorm.weight False cuda:0\n", "transformer.h.7.input_layernorm.bias False cuda:0\n", "transformer.h.7.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.7.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.7.self_attention.dense.weight False cuda:0\n", "transformer.h.7.self_attention.dense.bias False cuda:0\n", "transformer.h.7.post_attention_layernorm.weight False cuda:0\n", "transformer.h.7.post_attention_layernorm.bias False cuda:0\n", "transformer.h.7.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.7.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.7.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.7.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.8.input_layernorm.weight False cuda:0\n", "transformer.h.8.input_layernorm.bias False cuda:0\n", "transformer.h.8.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.8.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.8.self_attention.dense.weight False cuda:0\n", "transformer.h.8.self_attention.dense.bias False cuda:0\n", "transformer.h.8.post_attention_layernorm.weight False cuda:0\n", "transformer.h.8.post_attention_layernorm.bias False cuda:0\n", "transformer.h.8.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.8.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.8.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.8.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.9.input_layernorm.weight False cuda:0\n", "transformer.h.9.input_layernorm.bias False cuda:0\n", "transformer.h.9.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.9.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.9.self_attention.dense.weight False cuda:0\n", "transformer.h.9.self_attention.dense.bias False cuda:0\n", "transformer.h.9.post_attention_layernorm.weight False cuda:0\n", "transformer.h.9.post_attention_layernorm.bias False cuda:0\n", "transformer.h.9.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.9.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.9.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.9.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.10.input_layernorm.weight False cuda:0\n", "transformer.h.10.input_layernorm.bias False cuda:0\n", "transformer.h.10.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.10.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.10.self_attention.dense.weight False cuda:0\n", "transformer.h.10.self_attention.dense.bias False cuda:0\n", "transformer.h.10.post_attention_layernorm.weight False cuda:0\n", "transformer.h.10.post_attention_layernorm.bias False cuda:0\n", "transformer.h.10.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.10.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.10.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.10.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.11.input_layernorm.weight False cuda:0\n", "transformer.h.11.input_layernorm.bias False cuda:0\n", "transformer.h.11.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.11.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.11.self_attention.dense.weight False cuda:0\n", "transformer.h.11.self_attention.dense.bias False cuda:0\n", "transformer.h.11.post_attention_layernorm.weight False cuda:0\n", "transformer.h.11.post_attention_layernorm.bias False cuda:0\n", "transformer.h.11.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.11.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.11.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.11.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.12.input_layernorm.weight False cuda:0\n", "transformer.h.12.input_layernorm.bias False cuda:0\n", "transformer.h.12.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.12.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.12.self_attention.dense.weight False cuda:0\n", "transformer.h.12.self_attention.dense.bias False cuda:0\n", "transformer.h.12.post_attention_layernorm.weight False cuda:0\n", "transformer.h.12.post_attention_layernorm.bias False cuda:0\n", "transformer.h.12.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.12.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.12.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.12.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.13.input_layernorm.weight False cuda:0\n", "transformer.h.13.input_layernorm.bias False cuda:0\n", "transformer.h.13.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.13.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.13.self_attention.dense.weight False cuda:0\n", "transformer.h.13.self_attention.dense.bias False cuda:0\n", "transformer.h.13.post_attention_layernorm.weight False cuda:0\n", "transformer.h.13.post_attention_layernorm.bias False cuda:0\n", "transformer.h.13.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.13.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.13.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.13.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.14.input_layernorm.weight False cuda:0\n", "transformer.h.14.input_layernorm.bias False cuda:0\n", "transformer.h.14.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.14.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.14.self_attention.dense.weight False cuda:0\n", "transformer.h.14.self_attention.dense.bias False cuda:0\n", "transformer.h.14.post_attention_layernorm.weight False cuda:0\n", "transformer.h.14.post_attention_layernorm.bias False cuda:0\n", "transformer.h.14.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.14.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.14.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.14.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.15.input_layernorm.weight False cuda:0\n", "transformer.h.15.input_layernorm.bias False cuda:0\n", "transformer.h.15.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.15.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.15.self_attention.dense.weight False cuda:0\n", "transformer.h.15.self_attention.dense.bias False cuda:0\n", "transformer.h.15.post_attention_layernorm.weight False cuda:0\n", "transformer.h.15.post_attention_layernorm.bias False cuda:0\n", "transformer.h.15.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.15.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.15.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.15.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.16.input_layernorm.weight False cuda:0\n", "transformer.h.16.input_layernorm.bias False cuda:0\n", "transformer.h.16.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.16.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.16.self_attention.dense.weight False cuda:0\n", "transformer.h.16.self_attention.dense.bias False cuda:0\n", "transformer.h.16.post_attention_layernorm.weight False cuda:0\n", "transformer.h.16.post_attention_layernorm.bias False cuda:0\n", "transformer.h.16.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.16.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.16.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.16.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.17.input_layernorm.weight False cuda:0\n", "transformer.h.17.input_layernorm.bias False cuda:0\n", "transformer.h.17.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.17.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.17.self_attention.dense.weight False cuda:0\n", "transformer.h.17.self_attention.dense.bias False cuda:0\n", "transformer.h.17.post_attention_layernorm.weight False cuda:0\n", "transformer.h.17.post_attention_layernorm.bias False cuda:0\n", "transformer.h.17.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.17.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.17.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.17.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.18.input_layernorm.weight False cuda:0\n", "transformer.h.18.input_layernorm.bias False cuda:0\n", "transformer.h.18.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.18.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.18.self_attention.dense.weight False cuda:0\n", "transformer.h.18.self_attention.dense.bias False cuda:0\n", "transformer.h.18.post_attention_layernorm.weight False cuda:0\n", "transformer.h.18.post_attention_layernorm.bias False cuda:0\n", "transformer.h.18.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.18.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.18.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.18.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.19.input_layernorm.weight False cuda:0\n", "transformer.h.19.input_layernorm.bias False cuda:0\n", "transformer.h.19.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.19.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.19.self_attention.dense.weight False cuda:0\n", "transformer.h.19.self_attention.dense.bias False cuda:0\n", "transformer.h.19.post_attention_layernorm.weight False cuda:0\n", "transformer.h.19.post_attention_layernorm.bias False cuda:0\n", "transformer.h.19.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.19.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.19.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.19.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.20.input_layernorm.weight False cuda:0\n", "transformer.h.20.input_layernorm.bias False cuda:0\n", "transformer.h.20.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.20.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.20.self_attention.dense.weight False cuda:0\n", "transformer.h.20.self_attention.dense.bias False cuda:0\n", "transformer.h.20.post_attention_layernorm.weight False cuda:0\n", "transformer.h.20.post_attention_layernorm.bias False cuda:0\n", "transformer.h.20.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.20.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.20.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.20.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.21.input_layernorm.weight False cuda:0\n", "transformer.h.21.input_layernorm.bias False cuda:0\n", "transformer.h.21.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.21.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.21.self_attention.dense.weight False cuda:0\n", "transformer.h.21.self_attention.dense.bias False cuda:0\n", "transformer.h.21.post_attention_layernorm.weight False cuda:0\n", "transformer.h.21.post_attention_layernorm.bias False cuda:0\n", "transformer.h.21.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.21.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.21.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.21.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.22.input_layernorm.weight False cuda:0\n", "transformer.h.22.input_layernorm.bias False cuda:0\n", "transformer.h.22.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.22.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.22.self_attention.dense.weight False cuda:0\n", "transformer.h.22.self_attention.dense.bias False cuda:0\n", "transformer.h.22.post_attention_layernorm.weight False cuda:0\n", "transformer.h.22.post_attention_layernorm.bias False cuda:0\n", "transformer.h.22.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.22.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.22.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.22.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.23.input_layernorm.weight False cuda:0\n", "transformer.h.23.input_layernorm.bias False cuda:0\n", "transformer.h.23.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.23.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.23.self_attention.dense.weight False cuda:0\n", "transformer.h.23.self_attention.dense.bias False cuda:0\n", "transformer.h.23.post_attention_layernorm.weight False cuda:0\n", "transformer.h.23.post_attention_layernorm.bias False cuda:0\n", "transformer.h.23.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.23.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.23.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.23.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.24.input_layernorm.weight False cuda:0\n", "transformer.h.24.input_layernorm.bias False cuda:0\n", "transformer.h.24.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.24.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.24.self_attention.dense.weight False cuda:0\n", "transformer.h.24.self_attention.dense.bias False cuda:0\n", "transformer.h.24.post_attention_layernorm.weight False cuda:0\n", "transformer.h.24.post_attention_layernorm.bias False cuda:0\n", "transformer.h.24.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.24.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.24.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.24.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.25.input_layernorm.weight False cuda:0\n", "transformer.h.25.input_layernorm.bias False cuda:0\n", "transformer.h.25.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.25.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.25.self_attention.dense.weight False cuda:0\n", "transformer.h.25.self_attention.dense.bias False cuda:0\n", "transformer.h.25.post_attention_layernorm.weight False cuda:0\n", "transformer.h.25.post_attention_layernorm.bias False cuda:0\n", "transformer.h.25.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.25.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.25.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.25.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.26.input_layernorm.weight False cuda:0\n", "transformer.h.26.input_layernorm.bias False cuda:0\n", "transformer.h.26.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.26.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.26.self_attention.dense.weight False cuda:0\n", "transformer.h.26.self_attention.dense.bias False cuda:0\n", "transformer.h.26.post_attention_layernorm.weight False cuda:0\n", "transformer.h.26.post_attention_layernorm.bias False cuda:0\n", "transformer.h.26.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.26.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.26.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.26.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.27.input_layernorm.weight False cuda:0\n", "transformer.h.27.input_layernorm.bias False cuda:0\n", "transformer.h.27.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.27.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.27.self_attention.dense.weight False cuda:0\n", "transformer.h.27.self_attention.dense.bias False cuda:0\n", "transformer.h.27.post_attention_layernorm.weight False cuda:0\n", "transformer.h.27.post_attention_layernorm.bias False cuda:0\n", "transformer.h.27.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.27.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.27.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.27.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.28.input_layernorm.weight False cuda:0\n", "transformer.h.28.input_layernorm.bias False cuda:0\n", "transformer.h.28.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.28.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.28.self_attention.dense.weight False cuda:0\n", "transformer.h.28.self_attention.dense.bias False cuda:0\n", "transformer.h.28.post_attention_layernorm.weight False cuda:0\n", "transformer.h.28.post_attention_layernorm.bias False cuda:0\n", "transformer.h.28.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.28.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.28.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.28.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.h.29.input_layernorm.weight False cuda:0\n", "transformer.h.29.input_layernorm.bias False cuda:0\n", "transformer.h.29.self_attention.query_key_value.weight False cuda:0\n", "transformer.h.29.self_attention.query_key_value.bias False cuda:0\n", "transformer.h.29.self_attention.dense.weight False cuda:0\n", "transformer.h.29.self_attention.dense.bias False cuda:0\n", "transformer.h.29.post_attention_layernorm.weight False cuda:0\n", "transformer.h.29.post_attention_layernorm.bias False cuda:0\n", "transformer.h.29.mlp.dense_h_to_4h.weight False cuda:0\n", "transformer.h.29.mlp.dense_h_to_4h.bias False cuda:0\n", "transformer.h.29.mlp.dense_4h_to_h.weight False cuda:0\n", "transformer.h.29.mlp.dense_4h_to_h.bias False cuda:0\n", "transformer.ln_f.weight False cuda:0\n", "transformer.ln_f.bias False cuda:0\n", "transformer.prompt_embeddings.weight True cuda:0\n", "Parameter containing:\n", "tensor([[ 0.0289, 0.0230, 0.0049, ..., -0.0024, -0.0144, -0.0053],\n", " [-0.0165, 0.0022, 0.0458, ..., -0.0156, 0.0053, -0.0038],\n", " [ 0.0039, -0.0245, -0.0135, ..., -0.0011, 0.0008, 0.0165],\n", " ...,\n", " [-0.0042, 0.0283, 0.0045, ..., -0.0233, 0.0101, 0.0013],\n", " [-0.0258, -0.0271, 0.0120, ..., 0.0169, 0.0161, -0.0055],\n", " [ 0.0060, -0.0394, 0.0309, ..., 0.0285, -0.0300, 0.0055]],\n", " device='cuda:0', requires_grad=True)\n", "transformer.intermediate_prompt_embeddings.weight True cuda:0\n", "Parameter containing:\n", "tensor([[0., 0., 0., ..., 0., 0., 0.],\n", " [0., 0., 0., ..., 0., 0., 0.],\n", " [0., 0., 0., ..., 0., 0., 0.],\n", " ...,\n", " [0., 0., 0., ..., 0., 0., 0.],\n", " [0., 0., 0., ..., 0., 0., 0.],\n", " [0., 0., 0., ..., 0., 0., 0.]], device='cuda:0', requires_grad=True)\n", "score.weight True cuda:0\n", "Parameter containing:\n", "tensor([[ 0.0007, -0.0262, 0.0132, ..., 0.0213, -0.0057, -0.0152],\n", " [ 0.0161, 0.0150, -0.0190, ..., -0.0135, -0.0326, 0.0023]],\n", " device='cuda:0', requires_grad=True)\n" ] } ], "source": [ "model = BloomForSequenceClassification.from_pretrained(\n", " MODEL_NAME, \n", "# num_prefix_tokens=NUM_PREFIX_TOKENS, \n", " num_labels=NUM_LABELS\n", ").to(DEVICE)\n", " \n", "for name, p in model.named_parameters():\n", " if 'score' in name or 'prompt' in name:\n", " p.requires_grad = True\n", " else:\n", " p.requires_grad = False\n", "\n", "model.transformer.intermediate_prompt_embeddings.weight.data.zero_()\n", "print_params(model)" ] }, { "cell_type": "code", "execution_count": 4, "id": "95b03a47", "metadata": {}, "outputs": [], "source": [ "# params = dict(our_model.named_parameters())\n", "# for name, p in model.named_parameters():\n", "# print(f'Check: {name}')\n", "# assert torch.allclose(p, params[name]), name" ] }, { "cell_type": "markdown", "id": "901b50fd", "metadata": {}, "source": [ "## Dataset\n", "\n", "This examples operates on SST-2 dataset for binary sentence classification." ] }, { "cell_type": "code", "execution_count": 5, "id": "864fd3dc", "metadata": {}, "outputs": [], "source": [ "import random\n", "from IPython.display import display, HTML\n", "\n", "def show_random_elements(dataset, num_examples=10):\n", " assert num_examples <= len(dataset), \"Can't pick more elements than there are in the dataset.\"\n", " picks = []\n", " for _ in range(num_examples):\n", " pick = random.randint(0, len(dataset)-1)\n", " while pick in picks:\n", " pick = random.randint(0, len(dataset)-1)\n", " picks.append(pick)\n", " \n", " df = pd.DataFrame(dataset[picks])\n", " for column, typ in dataset.features.items():\n", " if isinstance(typ, datasets.ClassLabel):\n", " df[column] = df[column].transform(lambda i: typ.names[i])\n", " display(HTML(df.to_html()))" ] }, { "cell_type": "code", "execution_count": 6, "id": "18ca24c8", "metadata": {}, "outputs": [], "source": [ "GLUE_TASKS = [\"cola\", \"mnli\", \"mnli-mm\", \"mrpc\", \"qnli\", \"qqp\", \"rte\", \"sst2\", \"stsb\", \"wnli\"]\n", "task = 'sst2'" ] }, { "cell_type": "code", "execution_count": 7, "id": "5d110a5e", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Jul 26 05:43:35.435 [WARN] [datasets.builder.download_and_prepare:641] Reusing dataset glue (/home/dbaranchuk/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "4cadddcffa164eeaa7307bb1dcfba416", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/3 [00:00 of the transform datasets.arrow_dataset.Dataset._map_single couldn't be hashed properly, a random hash was used instead. Make sure your transforms and parameters are serializable with pickle or dill for the dataset fingerprinting and caching to work. If you reuse this transform, the caching mechanism will consider it to be different from the previous calls and recompute everything. This warning is only showed once. Subsequent hashing failures won't be showed.\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "e6d294129ce343f9bba3768a844c03de", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/68 [00:00" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Val accuracies: [0.9495412844036697, 0.9403669724770642, 0.9403669724770642, 0.944954128440367, 0.9495412844036697, 0.944954128440367]\n" ] } ], "source": [ "progress_bar = tqdm(range(num_training_steps))\n", "\n", "loss_history = []\n", "accuracy_history = []\n", "\n", "for epoch in range(num_epochs):\n", " for batch in train_dataloader:\n", " batch = {k: v.to(DEVICE) for k, v in batch.items()}\n", " \n", " model.train()\n", " outputs = model(**batch)\n", " loss = outputs.loss\n", " loss.backward()\n", " loss_history.append(loss.item())\n", "\n", " optimizer.step()\n", " lr_scheduler.step()\n", " optimizer.zero_grad()\n", " progress_bar.update(1)\n", " \n", " print(f'Iteration: {progress_bar.n}')\n", " plt.figure(figsize=[20, 8])\n", " plt.subplot(1,2,1)\n", " plt.title('Train loss over time', fontsize=12); plt.grid();\n", " plt.plot(moving_average(loss_history, span=10))\n", " plt.scatter(range(len(loss_history)), loss_history, alpha=0.1)\n", "\n", " accuracy = eval_metrics(model, valid_dataloader, device=DEVICE)\n", " accuracy_history.append(accuracy['accuracy'])\n", "\n", " plt.subplot(1,2,2)\n", " plt.title('Val accuracy', fontsize=12); plt.grid();\n", " plt.plot(accuracy_history)\n", " plt.show()\n", " print('Val accuracies: ', accuracy_history)" ] }, { "cell_type": "code", "execution_count": 15, "id": "197b3f4b", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Iteration: 10525\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAABHoAAAHeCAYAAADkemwqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAB9B0lEQVR4nOzdd3xV9eH/8fe5MzskAS4rLNmIigRBQahsBGRaW5UqrvanrasVR6sorlrFamtbpVq/1qodVkGGskRxMARRZA8ZYSVkJzfJnef3RyAlAhJGcnKS19OHD8i95577PvfehHvf+ZzPxzBN0xQAAAAAAABsz2F1AAAAAAAAAJwdFD0AAAAAAAD1BEUPAAAAAABAPUHRAwAAAAAAUE9Q9AAAAAAAANQTFD0AAAAAAAD1BEUPYDM33XST3n333dO67aBBg/T555+f5UR13+rVqzV8+HCrYwAAAKhz587avXu31TEA1GMUPUAt6NmzZ+X/Xbp00XnnnVf59XvvvXdK+3r55Zc1fvz4GkpaP3z3DVRGRoYWLFhgYSIAAFBf3HjjjXr++eePuXzx4sXq16+fwuGwBakA4H8oeoBasHbt2sr/W7RooRdffLHy6yuuuKJyO94YnBoeLwAAUNvGjx+v9957T6ZpVrn8vffe05gxY+RyuSxKduYikYjVEQCcBRQ9gIVWrlypAQMGaObMmerXr5/uv/9+FRYW6qc//an69u2r3r1766c//akOHjxYeZvJkyfrP//5jyTpnXfe0Y9//GM99dRT6t27twYNGqSPP/64WvcdDAb1+OOPq3///urfv78ef/xxBYNBSVJeXp5++tOfKiMjQxdddJGuvvpqRaNRSdLMmTN16aWXqmfPnho+fLiWL19+3P0XFxdr6tSp6tu3ry677DL9+c9/VjQaVTAYVEZGhrZu3Vq5bV5ens477zzl5uZKkpYuXaqxY8cqIyNDP/rRj7R58+bKbQcNGqSZM2dqzJgxuuCCC44pe6655hpJ0tixY9WzZ0/Nnz+/8nE+eh8vv/xy5T4eeOAB5eTk6KabblLPnj11/fXXq7CwsHL7r776Sj/60Y+UkZGhK664QitXrqzWYwwAAOqfIUOGqKCgQKtXr668rLCwUEuXLtW4ceO0bt06XXXVVcrIyFD//v01ffr0yvdYJ/Pf//5XI0eOVM+ePTV48GD985//rHL94sWLNXbsWF144YUaMmSIli1bJkkqKCjQ/fffr/79+6t379669dZbJf3vveLRjh75fN9992natGm6+eabdcEFF2jlypX66KOPNG7cOF144YUaOHCg/vjHP1a5/erVqyvfFw0cOFDvvPOO1q1bp0suuaRKUbRw4cIqv9AEUHsoegCL5eTkVL45ePTRRxWNRjVhwgQtXbpUS5culdfr1fTp0094+3Xr1qldu3ZasWKFbrrpJv36178+5jdMx/OXv/xFX3/9tWbPnq333ntP33zzjf785z9Lkl599VX5fD4tX75cn332me6++24ZhqFvv/1Wb7zxht5++22tXbtWr7zyilq2bHnc/T/66KMqLi7W4sWL9frrr2v27Nn673//K4/Ho6FDh2revHmV277//vvq3bu30tLStHHjRj3wwAOaPn26Vq5cqauuukq33nprlTdI8+bN08yZM7V69epjfmv2xhtvSJJmz56ttWvX6vLLLz9uvoULF+rVV1/VggULtHTpUt188826++67tWLFCkWjUb3++uuSpKysLP30pz/V//t//0+rVq3Svffeq9tvv115eXknfYwBAED9ExMTo5EjR2rWrFmVl73//vtq3769unTpIofDofvvv18rVqzQP//5Ty1fvlxvvvlmtfadlpaml156SV9++aWefPJJPfnkk9qwYYOkivd89957r6ZOnarVq1frjTfeqHwfNnXqVJWVlWnevHn6/PPPdf3111f7eObOnauf/exn+vLLL9WrVy/Fxsbqqaee0urVq/XSSy/prbfe0uLFiyVJ+/bt080336xrr71Wy5cv16xZs9S1a1edd955atSokT799NPK/c6ePVvjxo2rdg4AZw9FD2Axh8Oh22+/XR6PRzExMUpJSdHw4cMVGxurhIQE/b//9//0xRdfnPD2LVq00A9/+EM5nU6NHz9ehw4dUk5Ozknvd86cObrtttuUlpam1NRU3XbbbZXzBblcLh06dEj79++X2+1WRkaGDMOQ0+lUMBjUjh07FAqF1KpVK7Vu3fqYfUciEc2fP1+//OUvlZCQoFatWmnKlCmV+x8zZkyVomfOnDkaM2aMJOlf//qXrrrqKp1//vmVx+R2u/XVV19Vbj958mQ1b95cMTEx1XqMj+faa69V48aN5fP5lJGRofPOO0/dunWT1+vV0KFDtXHjRkkVb1IGDBiggQMHyuFwqF+/fjr33HOrPXIKAADUP+PGjdOCBQsUCAQkSbNmzaqcQ/Hcc8/VBRdcIJfLpVatWumqq6763vdyR/vBD36g1q1byzAMXXTRRerXr1/lyKG3335bEydOVL9+/eRwOOTz+XTOOecoOztby5Yt0yOPPKLk5GS53W5ddNFF1T6WwYMHq1evXnI4HPJ6verTp486d+4sh8OhLl26aNSoUVq1apWkilLokksu0ejRo+V2u5WSkqKuXbtWPiZH3usVFBTo008/1ejRo6udA8DZY98TSIF6IiUlRV6vt/LrsrIyPfnkk/rkk08qTx/y+/2KRCJyOp3H3L5x48aVf4+NjZUklZaWnvR+s7Oz1aJFi8qvW7RooezsbEkVkwy+8MILuuGGGyRJV111lW655Ra1adNGDzzwgP74xz9q+/bt6t+/v+677z75fL4q+87Pz1coFDpm/1lZWZKkPn36qLy8XF9//bXS0tK0efNmDRkyRJK0f/9+zZo1S//4xz8qbxsKhSqzSVLz5s1Penwnc/Tj5vV6q3wdExNT+Rju379fH3zwgZYuXVp5fTgcVp8+fc44AwAAsKeMjAylpKRo8eLF6tGjh7755hu98MILkqSdO3fqt7/9rdavX6+ysjJFIhF17969Wvv9+OOP9ac//Um7du1SNBpVeXm5OnXqJEk6cOCABg4ceMxtDh48qOTkZCUnJ5/WsXz3fdXXX3+tZ555Rtu2bVMoFFIwGNSIESMqMxzvl3xSxWnzI0eOVGlpqd5//31lZGSoadOmp5UJwJlhRA9gMcMwqnz9t7/9TTt37tS///1vffnll5WnIlXndKxT0bRpU+3fv7/y6wMHDlT+Y5yQkKD77rtPS5Ys0V/+8he9+uqrlXPxjBkzRm+99ZaWLl0qwzD0zDPPHLPvlJQUud3uY/Z/pBByOp0aMWKE5s6dq3nz5ukHP/iBEhISJFW82fjZz36m1atXV/7/9ddfV/mN0Hcfs5rUvHlzjR07tkqer776SrfcckutZQAAAHXP2LFjNWvWLL333nvq379/5S+NHn74YbVv314LFizQl19+qbvuuqta7+OCwaBuv/123XDDDfrss8+0evVqDRgwoPK2zZs31549e465XbNmzVRYWKiioqJjrouNjVV5eXnl14cOHTppjl/+8pcaPHiwPv74Y61Zs0Y/+tGPTppBknw+n3r27KmFCxdq9uzZzM8DWIiiB6hj/H6/vF6vkpKSVFBQUPnbobNt1KhR+stf/qK8vDzl5eXpT3/6U+XpU0uXLtXu3btlmqYSExPldDor5+hZvny5gsGgPB6PvF6vHI5jf4wcKXJ+//vfq6SkRPv27dOrr75a5R/8MWPG6P3339ecOXOqlDhXXnml/vnPf+rrr7+WaZoqLS3VRx99pJKSkmofW+PGjZWZmXkGj87/XHHFFVq6dKk++eQTRSIRBQIBrVy5ssoE2QAAoOEZN26cli9frn//+99V5qLx+/2Kj49XfHy8duzYobfeeqta+wsGgwoGg0pNTZXL5dLHH3+szz77rPL6SZMm6Z133tHy5csVjUaVlZWlHTt2qGnTphowYIAeeeQRFRYWKhQKVZ4q1qVLF23btk2bNm1SIBA4ZmLl4/H7/UpOTpbX69W6des0d+7cyuvGjBmjzz//XPPnz1c4HFZ+fr42bdpUef3YsWP1yiuvaOvWrRo2bFi1jhvA2UfRA9Qx1113nQKBgPr27aurrrpKl156aY3cz6233qpzzz1XV1xxha644gp17969coWG3bt3a8qUKerZs6euuuoq/fjHP1bfvn0VDAY1Y8YM9enTR/3791deXp7uvvvu4+7/wQcfVGxsrIYMGaKrr75ao0eP1sSJEyuvP//88xUbG6vs7OwqK2L16NFDjz76qKZPn67evXtr2LBheuedd07p2H7+85/rvvvuU0ZGhubPn38aj87/NG/eXH/+85/10ksv6eKLL9bAgQP1yiuvVK5CBgAAGqZWrVqpZ8+eKisr0+DBgysvv/feezV37lxdeOGFevDBB0+4MMR3JSQk6De/+Y3uvPNO9e7dW3PnztWgQYMqrz/vvPP05JNP6oknnlCvXr107bXXVo6e/t3vfieXy6WRI0fqkksu0WuvvSZJateunW677TZdf/31GjZsmHr16nXSHNOmTdMf/vAH9ezZU3/60580cuTIyutatGihv/71r3r11Vd10UUXady4cVVWRx06dKj27dunoUOHVk4pAKD2GebZPh8EAAAAANAgDRkyRNOnT9cll1xidRSgwWJEDwAAAADgjC1YsECGYahv375WRwEaNFbdAgAAAACckcmTJ2v79u363e9+d9w5HAHUHk7dAgAAAAAAqCeoWgEAAAAAAOqJGj11a82aNTW5ewAAUEdUZyUX1B7egwEAUP+d6P1Xjc/Rwxs/AADqN0qFuon3YAAA1F/f9/6LU7cAAAAAAADqCYoeAAAAAACAeoKiBwAAAAAAoJ6o1hw9gwYNUnx8vBwOh5xOp9555x0VFBTorrvu0r59+9SyZUs999xzSk5Orum8AAAAAAAAOIFqj+h57bXXNHv2bL3zzjuSpJkzZ+riiy/WwoULdfHFF2vmzJk1FhIAAAAAAAAnd9qnbi1ZskTjxo2TJI0bN06LFy8+W5kAAAAAAABwGqpd9Nx4442aMGGC/vWvf0mScnNz1bRpU0lSkyZNlJubWzMJAQAAAAAAUC3VmqPnrbfeks/nU25urqZMmaL27dtXud4wDBmGUSMBAQAAAAAAUD3VGtHj8/kkSWlpaRo6dKjWrVuntLQ0ZWdnS5Kys7OVmppacykBAAAAAABwUictekpLS1VSUlL5988++0wdO3bUoEGDNGvWLEnSrFmzNHjw4BoNCgAAAAAAgO930lO3cnNzddttt0mSIpGIRo8erQEDBqhHjx6688479fbbb6tFixZ67rnnajorAAAAAAAAvsdJi5709HS99957x1yekpKi1157rUZCAQAAAAAA4NSd9vLqAAAAAAAAqFsoegAAAAAAAOoJih4AAAAAAIB64qRz9NQ15aGIcksCCoSj8rocSkvwKsbttDoWAAAAAACA5Ww1oqc8FNG+/FJFTSnO41TUlPbll6o8FLE6GgAAAAAAgOVsVfTklgTkcTnlcTlkGIY8Loc8LqdySwJWRwMAAAAAAJAkZeaV6j+rMy25b1sVPYFwVG6nUeUyt9NQIBy1KBEAAAAAAMD/RKOmfvHWWr2wdLsl92+rosfrcigUMatcFoqY8rpsdRgAAAAAAKCeenftPn2VWaDbB3W05P5t1ZCkJXgVDEcUDEdlmqaC4aiC4YjSErxWRwMAAAAAAA1cSSCs336wWRekN9L4ni0tyWCroifG7VTLlDg5DKk0GJHDkFqmxLHqFgAAAAAAsNwfP9ymQ8UBPXxFdzkcxslvUANst7z6kbIHAAAAAACgrtiZ49ffPt2pSb1a6YL0RpblsNWIHgAAAAAAgLro0bkb5XU5NXVEZ0tzUPQAAAAAAACcgaVbsvXh5mzdPriDmibGWJqFogcAAAAAAOA0BcNRPTpno9o3jtf1l7SzOg5FDwAAAAAAwOn6v8936tscvx4c000el/U1i/UJAAAAAAAAbCi7uFx/WLJdg7o01WWdm1odRxJFDwAAAAAAwGl5+oMtCoQjenB0N6ujVKLoAQAAAAAAOEVfZRboP2v26ob+7dSucbzVcSpR9AAAAAAAAJyCaNTUw+9tUJNEr34xqKPVcaqg6AEAAAAAADgF767dp68yC3TviC5K8LqsjlMFRQ8AAAAAAEA1lQTC+u0Hm3VBeiNN6NnS6jjHqFu1EwAAAAAAQB32xw+36VBxQH/9SYYcDsPqOMdgRA8AAAAAAEA17Mzx62+f7tSkXq10QXojq+McF0UPAAAAAABANTw2d6O8LqemjuhsdZQTougBAAAAAAA4iaVbsrVkc7ZuH9xBTRNjrI5zQhQ9AAAAAAAA3yMYjurRORvVvnG8rr+kndVxvhdFDwAAAAAAwPd47fNd+jbHrwdHd5PHVberlLqdDgAAAAAAwELZxeV6fsk2DerSVJd1aWp1nJOi6AEAAAAAADiBpz/YokA4ogdHd7M6SrVQ9AAAAAAAABzHV5kF+s+avbqhXzu1axxvdZxqoegBAAAAAAD4jmjU1MPvbVCTRK9+PqiD1XGqjaIHAAAAAADgO95du09fZRbo3hFdlBjjtjpOtVH0AAAAAAAAHKUkENZvP9is89MbaULPllbHOSUuqwMAAAAAAADUJX/8cJsOFQf0159kyOEwrI5zShjRAwAAAAAAcNjOHL/+9ulOTerVShekN7I6zimj6AEAAAAAADjssbkb5XU5NXVEZ6ujnBaKHgAAAAAAAElLt2RryeZs/WJQBzVNjLE6zmmh6AEAAAAAAA1eMBzVo3M2qn3jeE3p187qOKeNogcAAAAAADR4r32+S9/m+PXg6G7yuOxbl9g3OQAAAAAAwFmQXVyu55ds02Wdm+iyLk2tjnNGKHoAAAAAAECD9vQHWxQIR/Tg6G5WRzljFD0AAAAAAKDB+jqzQP9Zs1c39Gun9k0SrI5zxih6AAAAAABAgxSNmnp4zgY1TvDq54M6WB3nrKDoAQAAAAAADdK7a/dp7Z4C3TeyixJj3FbHOSsoegAAAAAAQINTEgjrtx9s1vnpjTShZ0ur45w1LqsDAAAAAAAA1LYXPtyuQ8UBzZzcSw6HYXWcs4YRPQAAAAAAoEHZmePXK59+q0m9Wqln6xSr45xVFD0AAAA1ZNmyZRo+fLiGDh2qmTNnHnP9vn37dN1112nMmDGaPHmyDh48WHld165dNXbsWI0dO1Y/+9nPjrntY489pp49e9ZofgAA6qvH5m6U1+XU1BGdrY5y1nHqFgAAQA2IRCKaPn26Xn31Vfl8Pk2aNEmDBg1Shw7/W9Hjqaee0rhx4zR+/HgtX75cM2bM0NNPPy1JiomJ0ezZs4+772+++UaFhYW1chwAANQ3S7dka8nmbN0/souaJsZYHeesY0QPAABADVi3bp3atGmj9PR0eTwejRo1SkuWLKmyzY4dO9S3b19JUt++fY+5/ngikYh+97vf6Z577qmR3AAA1GfBcFSPztmodo3jNaVfO6vj1AiKHgAAgBqQlZWlZs2aVX7t8/mUlZVVZZsuXbpo4cKFkqRFixbJ7/crPz9fkhQIBDRhwgT98Ic/1OLFiytv849//EODBw9W06ZNa+EoAACoX177fJe+zfHrodHd5HHVz0qEU7cAAAAsMnXqVD366KN69913lZGRIZ/PJ6fTKUlaunSpfD6fMjMzdd1116lTp07yer364IMP9Prrr1ucHAAA+zlUHNAflmzTZZ2b6LIu9fcXJhQ9AAAANcDn81WZXDkrK0s+n++YbV544QVJkt/v18KFC5WUlFR5nSSlp6froosu0saNGxUTE6M9e/Zo2LBhkqSysjINHTpUixYtqo1DAgDA1p5esFnl4YgeHN3N6ig1qn6OUwIAALBYjx49tGvXLmVmZioYDGrevHkaNGhQlW3y8vIUjUYlSTNnztTEiRMlSYWFhQoGg5XbfPnll+rQoYN+8IMf6LPPPtOHH36oDz/8ULGxsZQ8AABUw9eZBfr36r26oV87tW+SYHWcGsWIHgAAgBrgcrn00EMP6aabblIkEtHEiRPVsWNHPf/88zr33HM1ePBgrVq1Ss8++6wMw1BGRoamTZsmqWKS5mnTpskwDJmmqZtvvrnKal0AAKD6olFTD8/ZoMYJXv18UP3/99QwTdOsqZ2vWbNGvXr1qqndAwCAOoB/7+senhMAAP7nnS/36u5/f62nJ52nKzPSrY5zVnzfv/WcugUAAAAAAOqlkkBYT76/WeenN9LEC1tZHadWcOoWAAAAAACol174cLsOFQc0c3IvORyG1XFqBSN6AAAAAABAvbMzx6+/fbpTEy9spZ6tU6yOU2soegAAAAAAQL3z2NyN8rgcundEZ6uj1CqKHgAAAAAAUK98tCVbSzZn6xeDOqhpUozVcWoVRQ8AAAAAAKg3guGops/dqHaN4zWlXzur49Q6ih4AAAAAAFBvvPb5Ln17yK8HR3eVx9Xwao+Gd8QAAAAAAKBeOlQc0B+WbNNlnZtoUBef1XEsQdEDAAAAAADqhacXbFZ5OKIHR3ezOoplKHoAAAAAAIDtfZ1ZoH+v3qsp/dqpfZMEq+NYhqIHAAAAAADYWjRq6uE5G9Q4watfDOpgdRxLUfQAAAAAAABbm/XVPq3dU6B7R3RWYozb6jiWougBAAAAAAC2VRII67fvb9b56Y008cJWVsexHEUPAAAAAACwrRc+3K7s4oAeHtNNDodhdRzLUfQAAAAAAABb2pXj198+3amJF7ZSz9YpVsepEyh6AAAAAACALT02b6PcTkP3juhsdZQ6o9pFTyQS0bhx4/TTn/5UkpSZmakrr7xSQ4cO1Z133qlgMFhjIQEAAAAAAI720ZZsLd6UrdsHd1TTpBir49QZ1S56/v73v+ucc86p/PqZZ57R9ddfr0WLFikpKUlvv/12jQQEAAAAAAA4WjAc1fS5G9Wucbym9GtndZw6pVpFz8GDB/XRRx9p0qRJkiTTNLVixQoNHz5ckjR+/HgtWbKk5lICAAAAAAAc9vflu/TtIb8eHN1VHhez0hytWo/GE088oXvuuUcOR8Xm+fn5SkpKksvlkiQ1a9ZMWVlZNZcSAAAAAABA0qHigJ5fvE0/6NxEg7r4rI5T55y06Fm6dKlSU1N17rnn1kYeAAAAAACAE3p6wWaVhyN6cHQ3q6PUSa6TbfDll1/qww8/1LJlyxQIBFRSUqLHH39cRUVFCofDcrlcOnjwoHw+WjQAAAAAAFBzvs4s0H/W7NXNl7bXOU0SrI5TJ510RM8vf/lLLVu2TB9++KGeffZZ9e3bVzNmzFCfPn20YMECSdK7776rQYMG1XhYAAAAAADQMEWjph6es0Fp8V79YlAHq+PUWac9Y9E999yjV199VUOHDlVBQYGuvPLKs5kLAAAAAACg0qyv9mntngLdO6KzEmPcVseps0566tbR+vTpoz59+kiS0tPTWVIdAAAAAADUuJJAWL99f7POT2+kiRe2sjpOncYaZAAAAAAAoE574cPtyi4O6OEx3eRwGFbHqdMoegAAAAAAQJ21K8evv326UxMvbKWerVOsjlPnUfQAAAAAAIA667F5G+V2Grp3RGero9gCRQ8AAAAAAKiTPtqSrcWbsvWLwR3VNCnG6ji2QNEDAAAAAADqnGA4qulzN6pd43hN6dfW6ji2QdEDAAAAAADqnL8v36VvD/n14Oiu8rqcVsexDYoeAAAAAABQpxwqDuj5xdv0g85NNKiLz+o4tkLRAwAAAAAA6pSnF2xWWSiiB0d3szqK7VD0AAAAAACAOmPd3gL9Z81e3dC/nc5pkmB1HNuh6AEAAAAAAHVCNGrq4fc2KC3eq18M6mB1HFui6AEAAAAAAHXCrK/26cs9BZo6orMSY9xWx7Elih4AAAAAAGC5kkBYv31/s85vlaxJF7ayOo5tuawOAAAAAAAA8Kel25VdHNBLk3vJ4TCsjmNbjOgBAAAAAACW2pXj1yuf7NSEC1uqZ+sUq+PYGkUPAAAAAACw1GPzNsrtNHTfiC5WR7E9ih4AAAAAAGCZj7Zka/GmbP1icEc1TYqxOo7tUfQAAAAAAABLBMNRTZ+7UW3T4jSlX1ur49QLFD0AAAAAAMASf1++S98e8uuhMd3kdTmtjlMvUPQAAAAAAIBad6g4oOcXb9MPOjfRoC4+q+PUGxQ9AAAAAACg1j29YLPKQhE9OLqb1VHqFYoeAAAAAABQq9btLdB/1uzVDf3b6ZwmCVbHqVcoegAAAAAAQK0xTVMPv7dBafFe/WJQB6vj1DsUPQAAAAAAoNbM+mqfvtxToKkjOisxxm11nHqHogcAAAAAANSKkkBYT87frPNbJWvSha2sjlMvUfQAAAAAAIBa8ael25VdHNC0K7rL4TCsjlMvUfQAAAAAAIAatyvHr1c+2akJF7bUha1TrI5Tb1H0AAAAAACAGvfYvI1yOw3dN6KL1VHqNYoeAAAAAABQoz7eekiLN2XrF4M7qmlSjNVx6jWKHgAAAAAAUGOC4agembNBbdPiNKVfW6vj1HsUPQAAAAAAoMb8ffkufXvIrwdHd5PX5bQ6Tr1H0QMAAAAAAGrEoeKAnl+8TT/o3ESDujS1Ok6DQNEDAAAAAABqxDMLtqgsFNGDo7vJMFhOvTZQ9AAAAAAAgLNu3d4C/XtNpqb0a6tzmiRYHafBoOgBAAAAAABnlWmaevi9DUqL9+r2wR2tjtOgUPQAAAAAAICzatZX+/TlngJNHdFZiTFuq+M0KC6rAwAAANRXy5Yt0+OPP65oNKorr7xSt9xyS5Xr9+3bpwceeEB5eXlq1KiRnn76aTVr1kyS1LVrV3Xq1EmS1Lx5c7344ouSpF/+8pdav3693G63evTooenTp8vt5g00AKDuKAmE9eT8zTq/VbImXdjK6jgNDiN6AAAAakAkEtH06dP18ssva968eZo7d662b99eZZunnnpK48aN05w5c3TrrbdqxowZldfFxMRo9uzZmj17dmXJI0lXXHGFPvjgA82ZM0eBQED/+c9/au2YAACojj8t3a7s4oCmXdFdDgcTMNc2ih4AAIAasG7dOrVp00bp6enyeDwaNWqUlixZUmWbHTt2qG/fvpKkvn37HnP98QwcOFCGYcgwDJ133nnKysqqkfwAAJyOXTl+vfLJTk24sKUubJ1idZwGiaIHAACgBmRlZVWehiVJPp/vmFKmS5cuWrhwoSRp0aJF8vv9ys/PlyQFAgFNmDBBP/zhD7V48eJj9h8KhTR79mxdeumlNXgUAACcmsfmbZLbaei+EV2sjtJgMUcPAACARaZOnapHH31U7777rjIyMuTz+eR0OiVJS5culc/nU2Zmpq677jp16tRJrVu3rrztI488ooyMDGVkZFgVHwCAKj7eekiLN2Xp3hFd1DQpxuo4DRZFDwAAQA3w+Xw6ePBg5ddZWVny+XzHbPPCCy9Ikvx+vxYuXKikpKTK6yQpPT1dF110kTZu3FhZ9LzwwgvKy8urvC0AAFYLRaKaPmeD2qbF6Yb+ba2O06Bx6hYAAEAN6NGjh3bt2qXMzEwFg0HNmzdPgwYNqrJNXl6eotGoJGnmzJmaOHGiJKmwsFDBYLBymy+//FIdOnSQJP3nP//Rp59+qmeffVYOB2/lAAB1w2uf79KOQ349OLqbvC6n1XEaNEb0AAAA1ACXy6WHHnpIN910kyKRiCZOnKiOHTvq+eef17nnnqvBgwdr1apVevbZZ2UYhjIyMjRt2jRJFZM0T5s2TYZhyDRN3XzzzZVFz7Rp09SiRQtdddVVkqShQ4fq5z//uWXHCQDAoeKAnl+8TQM7NdGgLk2tjtPgGaZpmjW18zVr1qhXr141tXsAAFAH8O993cNzAgCoTfe+vU7//XKvFtw1QOc0SbA6ToPwff/WM94XAAAAAACclnV7C/TvNZma0q8tJU8dQdEDAAAAAABOmWmaevi9DUqL9+gXgztaHQeHUfQAAAAAAIBTNuurffpyT4GmjuiipBi31XFwGEUPAAAAAAA4JSWBsJ6cv1nnt0rWpAtbWR0HR6HoAQAAAAAAp+RPS7cruzigaVd0l8NhWB0HR6HoAQAAAAAA1bYrx69XPtmpCT1b6sLWKVbHwXdQ9AAAAAAAgGp7bN4muZ2G7h3ZxeooOA6KHgAAAAAAUC0fbz2kxZuy9PNBHeVLirE6Do6DogcAAAAAAJxUKBLV9Dkb1DYtTjf0b2t1HJwARQ8AAAAAADip1z7fpR2H/HpwdDd5XU6r4+AEKHoAAAAAAMD3yikJ6PnF2zSwUxMN6tLU6jj4HhQ9AAAAAADgez39wRaVhSJ6cHQ3GQbLqddlFD0AAAAAAOCEvtlbqH+vydSUfm3VoWmC1XFwEhQ9AAAAAADguEzT1LT31ist3qNfDO5odRxUA0UPAAAAAAA4rllf7dOXewo0dXgXJcW4rY6DaqDoAQAAAAAAx/AHwvrt+5t1XqtkTerVyuo4qCaX1QEAAAAAAEDd86el25VVFNBfru0lh4MJmO2CET0AAAAAAKCKXTl+vfzJTk3o2VIXtk6xOg5OAUUPAAAAAACo4rF5m+RyGrp3ZBero+AUUfQAAAAAAIBKH289pMWbsvSLQR3lS4qxOg5OEUUPAAAAAACQJIUiUU2fs0Ft0+J0Q/+2VsfBaaDoAQAAAAAAkqTXPt+lHYf8+s2obvK6nFbHwWmg6AEAAAAAAMopCej5xds0sFMTDe7a1Oo4OE0UPQAAAAAAQE9/sEVloYgeHN1NhsFy6nZF0QMAAAAAQAP3zd5C/XtNpq6/pK06NE2wOg7OAEUPAAAAAAANmGmaenjOBqXFe3T7kI5Wx8EZougBAAAAAKABm/3Vfq3Zna+pw7soKcZtdRycIYoeAAAAAAAaKH8grCff36TzWiVrUq9WVsfBWUDRAwAAAABAA/WnpduVVRTQtDHd5XAwAXN94DrZBoFAQNdcc42CwaAikYiGDx+u22+/XZmZmbr77rtVUFCg7t2763e/+508Hk9tZAYAAAAAAGdod65fL3+yUxN6tlSvNilWx8FZctIRPR6PR6+99pree+89zZo1S5988om++uorPfPMM7r++uu1aNEiJSUl6e23366NvAAAAAAA4Cx4dO4muZyG7h3ZxeooOItOWvQYhqH4+HhJUjgcVjgclmEYWrFihYYPHy5JGj9+vJYsWVKzSY9SHoroP6szZZpmrd0nAAAAAAD1xbKth7R4U5Z+PqiDfEkxVsfBWVStOXoikYjGjh2rSy65RJdcconS09OVlJQkl6vizK9mzZopKyurRoMe7XcfbNE9b6/T0i3ZtXafAAAAAADUB6FIVI/M2aA2aXG6sX87q+PgLKtW0eN0OjV79mx9/PHHWrdunb799tuazvW9DpUEJEnF5WFLcwAAAAAAYDevfb5LOw759eCobvK6nFbHwVl2SqtuJSUlqU+fPvrqq69UVFSkcLiiaDl48KB8Pl+NBAQAAAAAAGdHTklAzy/epgGdmmhw16ZWx0ENOGnRk5eXp6KiIklSeXm5Pv/8c51zzjnq06ePFixYIEl69913NWjQoJpNehQWfAMAAAAA4NQ9s2CLykIRPTS6mwyDT9f10UmXV8/OztZ9992nSCQi0zQ1YsQIXXbZZerQoYPuuusuPffcc+ratauuvPLK2sgLAAAAAABOwzd7C/Wv1Zm6sV87dWiaYHUc1JCTFj1dunTRrFmzjrk8PT3d8iXVWXQLAAAAAICTM01TD8/ZoLR4j24f0tHqOKhBpzRHT13B6DIAAAAAAKpv9lf7tWZ3vqYO76KkGLfVcVCDbFn0AAAAAACA6vEHwnry/U06r1WyJvVqZXUc1DBbFz2mOHcLAAAAAIDv86el25VVFNC0Md3lcHCKTH1nu6KnPBRRWTAiScr3B1UeilicCAAAAACAuml3rl8vf7JTE3q2VK82KVbHQS2wVdFTHopoX35p5TieqCntyy+l7AEAAAAA4Dgem7dJLqehe0d2sToKaomtip7ckoA8Lqech2djdjoMeVxO5ZYELE4GAAAAAEDdsmzrIS3amKWfD+ogX1KM1XFQS2xV9ATCUbmdhnTUKYVup6FAOGpdKAAAAAAA6phQJKrpczeqTVqcbuzfzuo4qEW2Knq8LodCkaoTMIciprwuWx0GAAAAAAA16u/Ld2t7dokeHNVNXpfT6jioRbZqSNISvAqGI4pGK8qecMRUMBxRWoLX4mQAAAAAANQNOSUBPbdoqwZ0aqLBXZtaHQe1zFZFT4zbqZYpcTqyGpxhSC1T4hTjpp0EAAAAAECSnlmwRWWhiB4a3U2GwXLqDY2tih6pouyJ87gkSY3iPJQ8AAAAAAAc9s3eQv1rdaauv6StOjRNsDoOLGC7ogcAAAAAABzLNE09PGeD0uI9un1IR6vjwCK2LnoYgAYAAAAAQIXZX+3Xmt35umd4ZyXFuK2OA4vYuugxT74JAAAAAAD1nj8Q1pPvb9J5rZJ1Za90q+PAQvYsehjKAwAAAABApT8t3a6sooCmjekuh4MPzQ2ZPYseAAAAAAAgSdqd69fLn+zU+J4t1atNitVxYDFbFz2myclbAAAAAICG7bF5m+RyGrpvZBero6AOsGXRY3DuFgAAAAAAWrb1kBZtzNLPB3WQLynG6jioA2xZ9AAAAAAA0NCFIlFNn7tRbdLidGP/dlbHQR1h66KHE7cAAAAAAA3V35fv1vbsEv1mVDd5XU6r46COsGXRY3DmFgAAAACgAcspCei5xVs1oFMTDena1Oo4qENsWfRUYkgPAAAAAKABembBFpUFI3podDcZjIbAUWxZ9PASBgAAdrBs2TINHz5cQ4cO1cyZM4+5ft++fbruuus0ZswYTZ48WQcPHqy8rmvXrho7dqzGjh2rn/3sZ5WXZ2Zm6sorr9TQoUN15513KhgM1sqxAADqjm/2FupfqzN13SVt1aFpgtVxUMfYsugBAACo6yKRiKZPn66XX35Z8+bN09y5c7V9+/Yq2zz11FMaN26c5syZo1tvvVUzZsyovC4mJkazZ8/W7Nmz9eKLL1Ze/swzz+j666/XokWLlJSUpLfffrvWjgkAYD3TNPXwnA1Ki/fojiEdrY6DOsjWRY/JuVsAAKCOWrdundq0aaP09HR5PB6NGjVKS5YsqbLNjh071LdvX0lS3759j7n+u0zT1IoVKzR8+HBJ0vjx4096GwBA/TL7q/1asztf9wzvrKQYt9VxUAfZsujh9EMAAFDXZWVlqVmzZpVf+3w+ZWVlVdmmS5cuWrhwoSRp0aJF8vv9ys/PlyQFAgFNmDBBP/zhD7V48WJJUn5+vpKSkuRyuSRJzZo1O2afAID6yx8I68n3N6lHy2Rd2Svd6jioo1xWBwAAAGiopk6dqkcffVTvvvuuMjIy5PP55HRWLI+7dOlS+Xw+ZWZm6rrrrlOnTp2UkMA8DADQkP35o+3KKgroz9dcKIeDERA4PooeAACAGuDz+apMrpyVlSWfz3fMNi+88IIkye/3a+HChUpKSqq8TpLS09N10UUXaePGjRo+fLiKiooUDoflcrl08ODBY/YJAKifduf69ddlOzW+Z0v1apNqdRzUYbY8dQsAAKCu69Gjh3bt2qXMzEwFg0HNmzdPgwYNqrJNXl6eotGoJGnmzJmaOHGiJKmwsLByNa28vDx9+eWX6tChgwzDUJ8+fbRgwQJJ0rvvvnvMPgEA9dNj8zbJ5TR038guVkdBHceIHgAAgBrgcrn00EMP6aabblIkEtHEiRPVsWNHPf/88zr33HM1ePBgrVq1Ss8++6wMw1BGRoamTZsmqWKS5mnTpskwDJmmqZtvvlkdOnSQJN1zzz2666679Nxzz6lr16668sorrTxMAEAtWLb1kBZtzNLUEZ3lS4qxOg7qOFsXPSaLbgEAgDps4MCBGjhwYJXL7rjjjsq/jxgxQiNGjDjmdhdeeKHmzJlz3H2mp6ezpDoANCChSFTT525Um7Q43di/ndVxYAO2PHXLEJNOAQAAAADqv78v363t2SX6zahu8rqcVseBDdiy6DmCAT0AAAAAgPoqpySg5xZv1aUdG2tI16ZWx4FN2LLoMRjQAwAAAACo555ZsEVlwYimjekmgw/CqCZbFj0AAAAAANRn3+wt1L9WZ+q6S9qqQ9NEq+PARmxd9DAZMwAAAACgvjFNUw/P2aDUOI9uH9zR6jiwGVsWPYxYAwAAAADUV7O/2q81u/M1dURnJce6rY4Dm7Fl0QMAAAAAQH3kD4T15Pub1KNlsq7slW51HNiQy+oAZ8Jk3S0AAAAAQD3y54+2K6sooD9fc6EcDk5nwamz6YgeXuwAAAAAgPpld65ff122U+N7tlSvNqlWx4FN2bToAQAAAACgfnls3ia5nIbuG9nF6iiwMVsXPay6BQAAAACoDz7ZdkiLNmbptss6yJcUY3Uc2Jgtix5W3QIAAAAA1BehSFSPzNmoNmlxurF/O6vjwOZsWfQcwYAeAAAAAIDd/X35bm3PLtFvRnVTjNtpdRzYnK2LHgAAAAAA7Cy3JKDnFm/VpR0ba0jXplbHQT1g66KHM7gAAAAAAHb2zMItKgtGNG1MNxnMU4KzwNZFD6duAQAAAADs6pu9hfrnF5m67pK26tA00eo4qCdsWfTQcQIAAAAA7Mw0TT0yZ4NS4zy6fXBHq+OgHrFl0QMAAAAAgJ299/V+rd6dr3uGd1ZyrNvqOKhH7F30mJy8BQAAAACwF38grCfmb1KPlsm6MiPd6jioZ1xWBzgdzE8FAAAAALCrP3+0XVlFAf35mgvldPABF2eXvUf0AAAAAABgI3tyS/XXT3Zq3AUt1KtNqtVxUA/ZuujhxC0AAAAAgJ08Nm+jXA5D943sanUU1FO2LHoM1t0CAAAAANjMJ9sOaeHGLN12WQc1S46xOg7qKVsWPUcwFzMAAAAAwA5CkagembNRrVPjdGP/dlbHQT1my6KHyZgBAAAAAHby+vLd2p5dogdHd1OM22l1HNRjtix6AAAAAACwi9ySgH6/eKsu7dhYQ7o2tToO6jlbFz0m524BAAAAAOq4ZxZuUVkwomljusngFBXUMFsWPUe+Lah5AAAAAAB12fp9hfrnF5n6ycVt1aFpotVx0ADYs+g53IAyoAcAAAAAUFeZpqmH39ug1DiP7hjS0eo4aCBsWfQcQc8DAAAAAKir3vt6v1bvztc9wzsrOdZtdRw0ELYsejilEQAAAABQl5UGw3py/mad2zJJV2akWx0HDYjL6gBngsmYAQAAAAB10Z+X7tDBonK9cHVPOR2MVkDtseeIHvFNAgAAAACom/bklmrmJ99q3AUtlNE21eo4aGDsWfQc7nkY0AMAAAAAqGsem7dRLoeh+0Z2tToKGiB7Fj2H/zSZjhkAAAAAUId8su2QFm7M0m2XdVCz5Bir46ABsmfRw4geAAAAAEAdE4pE9cicjWqdGqcb+7ezOg4aKJsWPRVNDz0PAAAAAKCueH35bm3PLtFvRnVVjNtpdRw0UPYseg7/yYgeAAAAAEBdkFsS0O8Xb9WlHRtraDef1XHQgNmy6DnS9DBHDwAAAACgLnhm4RaVBSOaNqZb5VkogBVsWfQcWV6dET0AAAAAAKut31eof36RqZ9c3FYdmiZaHQcNnD2LHspRAAAAAEAdYJqmHn5vg1LjPLpjSEer4wAnL3oOHDigyZMn6/LLL9eoUaP02muvSZIKCgo0ZcoUDRs2TFOmTFFhYWGNhz3if3P0MKQHAAAAAGCd977er9W783XP8M5KjnVbHQc4edHjdDp13333af78+frXv/6lN998U9u3b9fMmTN18cUXa+HChbr44os1c+bM2sgrieXVAQAAAADWKw2G9eT8zTq3ZZKuzEi3Og4gqRpFT9OmTdW9e3dJUkJCgtq3b6+srCwtWbJE48aNkySNGzdOixcvrtGgR6uco6fW7hEAAAAAgKr+vHSHDhaV6+Ex3eV0MMcI6oZTmqNn79692rRpk84//3zl5uaqadOmkqQmTZooNze3RgIeDyN6AAAAAABW2pNbqpmffKtxF7RQRttUq+MAlapd9Pj9ft1+++164IEHlJCQUOU6wzBqdfm4yjl6GNMDAAAAALDAY/M2yuUwdN/IrlZHAaqoVtETCoV0++23a8yYMRo2bJgkKS0tTdnZ2ZKk7OxspabWYoNpsLw6AAAAAMAan27L0cKNWbrtsg5qlhxjdRygipMWPaZp6te//rXat2+vKVOmVF4+aNAgzZo1S5I0a9YsDR48uMZCfhdnPgIAAAAArBCKRPXInA1qnRqnG/u3szoOcIyTFj1r1qzR7NmztWLFCo0dO1Zjx47Vxx9/rFtuuUWfffaZhg0bps8//1y33HJLbeStggE9AAAAAIDa9Pry3dqWXaLfjOqqGLfT6jjAMVwn2yAjI0Nbtmw57nWvvfbaWQ9UHZXTAXHuFgAAAACgluSWBPT7xVt1acfGGtrNZ3Uc4LhOadWtuoLl1QEAAAAAte2ZhVtVFoxo2phutbogEXAq7Fn0sLw6AAAAAKAWrd9XqH9+sUc/ubitOjRNtDoOcEL2LHoO/8ny6gAAAACAmmaaph5+b4NS4zy6Y0hHq+MA38ueRQ8jegAAAAAAteS9r/dr9e58/Wp4ZyXHuq2OA3wvmxY9zNEDAAAAAKh5pcGwnpy/Wee2TNIPM9KtjgOc1ElX3arLGNEDAAAAAKhJf166QweLyvXC1T3ldDABM+o+m47oqfiTOXoAAAAAADVlT26pZn7yrcZe0EIZbVOtjgNUiz2LHlU2PQAAAAAA1IjH52+Uy2Ho/pFdrY4CVJs9ix56HgAAAABADfp0W44WbMjSbZd1ULPkGKvjANVmz6Ln8J8mk/QAAAAAAM6yUCSqR+ZsUOvUON3Yv53VcYBTYs+ih+XVAQCADSxbtkzDhw/X0KFDNXPmzGOu37dvn6677jqNGTNGkydP1sGDB6tcX1JSogEDBmj69OmVl82dO1djxozRmDFjdOONNyovL6/GjwMAGpp/rNitbdkl+s2oropxO62OA5wSWxY9AAAAdV0kEtH06dP18ssva968eZo7d662b99eZZunnnpK48aN05w5c3TrrbdqxowZVa5/7rnn1Lt378qvw+GwHn/8cb322muaM2eOOnfurDfeeKNWjgcAGorckoCeXbRVl3ZsrKHdfFbHAU6ZLYueI5MxM6AHAADUVevWrVObNm2Unp4uj8ejUaNGacmSJVW22bFjh/r27StJ6tu3b5Xr169fr9zcXPXr16/yMtM0ZZqmysrKZJqmSkpK1LRp09o5IABoIJ5ZuFWlwYgeGt1NhsFy6rAfexY9fK8BAIA6LisrS82aNav82ufzKSsrq8o2Xbp00cKFCyVJixYtkt/vV35+vqLRqJ566inde++9VbZ3u916+OGHNWbMGF166aXasWOHJk2aVPMHAwANxPp9hfrnF3t03cVt1dGXaHUc4LTYsug5gjl6AACAnU2dOlVffPGFxo0bp1WrVsnn88npdOrNN9/UgAEDqhRFkhQKhfTWW29p1qxZ+uSTT9S5c2e99NJLFqUHgPrFNE09/N4GpcZ5dMeQjlbHAU6by+oAZ8Lk5C0AAFBH+Xy+KpMrZ2VlyefzHbPNCy+8IEny+/1auHChkpKStHbtWq1Zs0ZvvfWW/H6/QqGQ4uLiNGzYMElS69atJUkjR4487iTPAIBT997X+7V6d76enNBDybFuq+MAp82WRc+R8yQZ0QMAAOqqHj16aNeuXcrMzJTP59O8efOOmWw5Ly9PjRo1ksPh0MyZMzVx4kRJqrLdO++8o/Xr1+tXv/qVsrKytGPHDuXl5Sk1NVWfffaZzjnnnFo9LgCoj0qDYT05f7O6t0jSDzPSrY4DnBF7Fj1WBwAAADgJl8ulhx56SDfddJMikYgmTpyojh076vnnn9e5556rwYMHa9WqVXr22WdlGIYyMjI0bdq0792nz+fTbbfdpmuuuUYul0stW7bUk08+WUtHBAD1118+2qGDReV64eqecjr4xAl7s2fRc/j7zmRIDwAAqMMGDhyogQMHVrnsjjvuqPz7iBEjNGLEiO/dx4QJEzRhwoTKr3/84x/rxz/+8dkNCgAN2J7cUr207FuNvaCFMtqmWh0HOGO2nIz5SL9KzQMAAAAAOBOPz98op2HovpFdrI4CnBX2LHqYowcAAAAAcIY+3ZajBRuy9PNBHdQ8OdbqOMBZYdOip+JPVt0CAAAAAJyOUCSqR+ZsUOvUON3Yv53VcYCzxpZFTzhSUfAUloW0L79U5aGIxYkAAAAAAHbyjxW7tS27RL8e1VUxbqfVcYCzxnZFT3koosKyoCTJ5TAUNUXZAwAAAACottySgH6/aKsu7dhYw7r5rI4DnFW2K3pySwJyOSpim6bkcTnkcTmVWxKwOBkAAAAAwA6eWbhV/mBED43uVjkHLFBf2K7oCYSjcjkPT8Z8+DK301AgHLUuFAAAAADAFtbvK9Q/v9ijn1zcRh19iVbHAc462xU9XpdD0e90OqGIKa/LdocCAAAAAKhFpmnqkTkblBLn0Z1DOlkdB6gRtmtH0hK8ihxueqKmqWA4qmA4orQEr8XJAAAAAAB12Xtf79cXu/J1z/DOSo51Wx0HqBG2K3pi3E6lxHskVay+5TCklilxzJIOAAAAADih0mBYT87frO4tkvTDjHSr4wA1xmV1gNPhOXyaVoLXpZYpcRanAQAAAADUdX/5aIcOFpXrj1f3lNPBBMyov2w3okeSDPFNCQAAAAConsy8Ur207FuNvaCFerdNtToOUKNsWfQcYZ58EwAAAABAA/fYvI1yGobuG9nF6ihAjbNn0XN4QI9J0wMAAAAA+B6fbc/Rgg1Zuu2yc9Q8OdbqOECNs2XRc+TELZMxPQAAAACAEwhFonpkzgalp8bqpkvbWx0HqBX2LHqMI0N6rM0BAAAAAKi7/rFit7Zmleg3o7qxUjMaDHsWPYf/pOcBAAAAABxPbklAv1+0Vf07NNawbj6r4wC1xp5FT+UcPVQ9AAAAAIBjzVi0Vf5gRNPGdPvfWSFAA2DvosfaGAAAAACAOmj9vkK9tWqPfnJxG3X0JVodB6hV9ix6Dp+8xYAeAACA2jPn6/369bvfKBCOWB0FAE7INE09MmeDUuI8unNIJ6vjALXOlkXPEfQ8AAAAtcftdOiNlXt02xtrFQxHrY4DAMc1Z90BfbErX/cM76zkWLfVcYBaZ++ihyE9AAAAtWbEuc306NjuWrwpS3f8c63CEcoeAHVLaTCsJ+dvUvcWSfphRrrVcQBL2LvosToAAABAAzP54rb6zaiuen/9Qd39768VifKODEDd8ZePduhAYbkevqK7nA4mYEbD5LI6wBnhfQUAAECtu+nS9gpGovrdB1vkcTn0u4nnycEHKgAWy8wr1UvLvtUV57dQ77apVscBLGProsek6QEAALDErT/ooGA4qucWb5Pb6dAT489l+WIAlnp83iY5DUP3X97F6iiApexd9NDzAAAAWOaOwR0VDEf15492yOtyaNqYbpQ9ACzx2fYcfbDhoH41rJOaJ8daHQewFEUPAAAATothGLpneGcFw1G9/OlOeVwO3T+yC2UPgFoVjkT1yJwNSk+N1U2Xtrc6DmA5exc9nLoFAABgKcMw9OtRXRWKRDVz2bfyOB361fDOVscC0ID8Y8Vubc0q0UuTeynG7bQ6DmA5WxY9FDwAAAB1h2EYmjamu4KRqF5Yul0el0O3D+5odSwADUBuSUDPLtqq/h0aa1g3n9VxgDrBlkXPEZy6BQAAUDc4HIYeH9dDgXBUzy7aKo/LoZ8NPMfqWADquRmLtsofjDBHGHAUexc9VgcAAABAJYfD0NOTzlcoYuq372+W2+nQjf3bWR0LQD21fl+h3lq1R9df0lYdfYlWxwHqDHsXPTQ9AAAAdYrTYejZH56vUDiqR+dulMfl0OS+bayOBaCeMU1Tj8zZoJQ4j+4c0snqOECd4rA6wJmh6QEAAKhr3E6H/vDjnhrStakenLVe//4i0+pIAOqZOesO6Itd+frVsM5KjnVbHQeoU2xd9DCiBwAAoG7yuBz60zUXakCnJrr3nXV6d+1eqyMBqCdKg2E9OX+TurdI0lW9062OA9Q59i56rA4AAACAE/K6nJo5uZf6tkvTL//9teau2291JAD1wIsf7dCBwnI9fEV3OR1MwAx8l72LHob0AAAA1GkxbqdeuT5Dvdqk6I5/fqUFGw5aHQmAjWXmlerFZd/qivNbqHfbVKvjAHWSvYseqwMAAADgpOI8Lv3t+t7q0TJZP3/zS324OcvqSABs6vF5m+Q0DN1/eRerowB1lr2LHpoeAAAAW0iMceu1Gy5Sl2ZJ+tk/vtQn2w5ZHQmAzXy2PUcfbDio2y47R82TY62OA9RZ9i56rA4AAACAakuOdev1Gy9S+8bxuvnvq7Xi21yrIwGwiXAkqkfmbFB6aqxuurS91XGAOs3eRQ9DegAAAGylUZxHb9zUR+kpcbrh/77Q6l15VkcCYAP/WLFbW7NK9OvLuynG7bQ6DlCn2broAQAAgP2kJXj1xs191CwpRte/+oW+yiywOhKAOizPH9Szi7aqf4fGGt7dZ3UcoM6zddHDgB4AAAB7apoYozdv7qvUeI9+8spKrd9XaHUkAHXUMwu3yB+MaNqYbjIMllMHTsbeRQ+z9AAAANhWs+QYvXlzHyXGuHXtKyu1+WCR1ZEA1DEb9hfqrVV7NLlvG3X0JVodB7AFexc99DwAAAC21iolTm/e3EcxLqeu+etKbc8utjoSgDrCNE098t5GpcR5dNeQTlbHAWyDogcAAACWapMWrzdu7iPDMHT1X1dqZ47f6kgA6oC56w5o1a48/WpYZyXHua2OA9iGLYseCh4AAID65ZwmCXrz5j4KR01d/dcVyswrtToSAAuVBsN6Yv4mdW+RpKt6p1sdB7AVWxY9RzBHDwAAQP3RyZeof9zYR6XBiH781xXaV1BmdSQAFnnxox06UFiuh6/oLqeDCZiBU2HvooeeBwAAoF7p1iJJ/7ixjwrLQrr6ryuUVVRudSQAtSwzr1QvLvtWV5zfQr3bplodB7Adexc9VgcAAADAWdejVbJeu+Ei5RQHdPVfV+hQccDqSABq0ePzNslpGLr/8i5WRwFsydZFD00PAABA/XRh6xS9OuUi7S8o17Uvr1SeP2h1JAC14PPtOfpgw0Hd+oNz1Dw51uo4gC3Zuuhhjh4AAID666J2qXrlugztyvXr2pdXqrA0ZHUkADUoHInq4TkblJ4aq5sHtLc6DmBb9i566HkAAADqtUs6NNbMn2Roe3aJfvK3lSoqp+wB6qt/rNitrVkl+vXl3RTjdlodB7Atexc9VgcAAABAjRvYqYn+fM2F2rC/SFNe/UIlgbDVkQCcZXn+oJ5dtFX9OqRpeHef1XEAW7N30cOQHgAAgAZhSDefXri6p77KLNCN//eFyoIRqyMBOItmLNwifzCiaWO6yzBYTh04Eycteu6//35dfPHFGj16dOVlBQUFmjJlioYNG6YpU6aosLCwRkOeCDUPAABAwzHi3Ob6/VUX6Itdebr576tVHqLsAeqDDfsL9eaqPZrct406+RKtjgPY3kmLngkTJujll1+uctnMmTN18cUXa+HChbr44os1c+bMGgv4fRjQAwAA0LBccX4L/W7S+fpsR45+9o81CoQpewA7M01Tj7y3UY1i3bprSCer4wD1wkmLnt69eys5ObnKZUuWLNG4ceMkSePGjdPixYtrJNyJhCJRSVJ5KKJ9+aX8NgcAAKABmdSrlZ4Y30MfbTmkn7+5tvK9IQD7mbvugFbtytM9w7soOc5tdRygXjitOXpyc3PVtGlTSVKTJk2Um5t7VkN9n/JQRPn+oCTJYUhRU5Q9AAAADcyPL2qt6WO7a9HGLN35z68UpuwBbKc0GNaT8zepe4skXdU73eo4QL3hOtMdGIZRq5Nl5ZYE5HRW9FOmJI/LUXl5y5S4WssBAAAAa/3k4rYKhqN6bN4muZ2GZvzwAjkdTOIK2MWLH+3Q/sJyPfejnnzvAmfRaRU9aWlpys7OVtOmTZWdna3U1NSzneuEAuGoXId/BhyZo8ftNFTKygsAAAANzk2XtlcgHNXTC7bI7XToqYnnycEHRqDOy8wr1UvLvtUV57fQRe1q7/Mk0BCc1qlbgwYN0qxZsyRJs2bN0uDBg89mpu/ldTkUPlzwHJmLORQx5XXZeqV4AABQDy1btkzDhw/X0KFDj7t4xb59+3TddddpzJgxmjx5sg4ePFjl+pKSEg0YMEDTp0+vvCwYDOrBBx/U8OHDNWLECC1YsKDGj6Ouu+2yDrpjcEf9Z81ePTh7vUxW7ADqvCfmb5LDMHT/5V2sjgLUOycd0XP33Xdr1apVys/P14ABA/SLX/xCt9xyi+688069/fbbatGihZ577rlaiFohLcGryOFzsE3TVDAcVTAc4bQtAABQp0QiEU2fPl2vvvqqfD6fJk2apEGDBqlDhw6V2zz11FMaN26cxo8fr+XLl2vGjBl6+umnK69/7rnn1Lt37yr7ffHFF5WamqoFCxYoGo2qoKCgtg6pTrtzSEcFI1H95aMdcjsdmjamW61OLwCg+j7fnqP31x/UL4d2UvPkWKvjAPXOSYueZ5999riXv/baa2c9THXEuJ1qFOeRVDERs8OQWqbEKcbttCQPAADA8axbt05t2rRRenrFBKOjRo3SkiVLqhQ9O3bs0P333y9J6tu3r2677bbK69avX6/c3FxdeumlWr9+feXl//3vf/X+++9LkhwOR62eQl+XGYahqcM7KxCK6m+f7ZTX5dB9I7tQ9gB1TDgS1SNzNio9NVY3D2hvdRygXrL3+U6MygUAAHVUVlaWmjVrVvm1z+dTVlZWlW26dOmihQsXSpIWLVokv9+v/Px8RaNRPfXUU7r33nurbF9UVCRJev755zV+/HjdfvvtysnJqeEjsQ/DMPTg6K6a3LeNXlr2rX6/aKvVkQB8xxsr92hLVrF+fXk3flkP1BDbFT3loYjySyuWVzdYXh0AANjY1KlT9cUXX2jcuHFatWqVfD6fnE6n3nzzTQ0YMKBKUSRJ4XBYBw8eVM+ePfXuu++qZ8+eeuqppyxKXzcZhqFHruiuH/VO1x8+3K4XPtxmdSQAh+X5g5qxcIv6dUjT8O4+q+MA9dYZL69e23JLAnI5WF4dAADUbT6fr8rkyllZWfL5fMds88ILL0iS/H6/Fi5cqKSkJK1du1Zr1qzRW2+9Jb/fr1AopLi4OP3yl79UbGyshg0bJkkaMWKE3n777do7KJtwOAw9Mb6HguGonlm4VR6XQ7cMOMfqWECDN2PhFvmDEU0b053TKoEaZLuiJxCOynlkHBLLqwMAgDqqR48e2rVrlzIzM+Xz+TRv3jzNmDGjyjZ5eXlq1KiRHA6HZs6cqYkTJ0pSle3eeecdrV+/Xr/61a8kSZdddplWrlypiy++WMuXL9c551BgHI/DYeh3k85TMBLVE/M3y+10aEq/dlbHAhqsDfsL9daqPfrJxW3VyZdodRygXrNd0eN1OXR40S2Zh5sellcHAAB1jcvl0kMPPaSbbrpJkUhEEydOVMeOHfX888/r3HPP1eDBg7Vq1So9++yzMgxDGRkZmjZt2kn3+6tf/UpTp07VE088odTUVD355JO1cDT25HI69PurLlDo8OSvHpdD1/RpY3UsoMExTVOPvLdRybFu3TWkk9VxgHrPdkVPWoJX4Wi08muWVwcAAHXVwIEDNXDgwCqX3XHHHZV/HzFihEaMGPG9+5gwYYImTJhQ+XXLli31xhtvnN2g9Zjb6dAff3yhfvaPNfr1u+vldjr0w4x0q2MBDcrcdQe0aleeHh9/rpLj3FbHAeo92w2DiXE7lXJ4efVI1GR5dQAAAHwvj8uhP19zoS7t2Fj3/nedZn+1z+pIQINRFozoyfmb1K15kn7Uu7XVcYAGwXZFj1Txm5kjf1LyAAAA4GRi3E7NnJyhvu3SdPe/v9b8bw5YHQloEP7y8Q7tLyzXw1d0l9PBBMxAbbBl0XOEaXUAAAAA2Easx6mXr8tQz/RGuv2ttVq44eDJbwTgtGXmleqlj3dozPktdFG7VKvjAA2GvYsek6oHAAAA1RfvdenVKb11bstk3fbml1q6JdvqSEC99cT8TXIYhu4f2cXqKECDYu+ix+oAAAAAsJ3EGLdeu+EidW6WqJ++vkafbsuxOhJQ73y+PUfvrz+oW39wjlo0irU6DtCg2LLoCR1eXz0YjmpffqnKQxGLEwEAAMBOkmPdev2GPmrfOF43/f0Lrfg21+pIQL0RjkT1yJyNapUSq5sHtLc6DtDg2K7oKQ9FlF8alCQZkqKmKHsAAABwylLiPfrHTX3UKiVON/zfF1qzO8/qSEC98MbKPdqSVazfjOrGwjmABWxX9OwvKJM/EJYkhaOmoqYpj8up3JKAxckAAABgN40TvHrzpj7yJcXo+r99oa8zC6yOBNhanj+oZxdtVb8OaRre3Wd1HKBBslXRUx6KaE+uX4YqluUzTelgYZki0agC4ajF6QAAAGBHTZNi9ObNfdQo3q3Jr6zUhv2FVkcCbGvGwi0qCYQ1bUx3GQbLqQNWsFXRk1sSUHyMS47DqQ1D8jidOlQckNdlq0MBAABAHdI8OVZv3tRXiTFuXfvySm05WGx1JMB2Nu4v0lur9mhy3zbq5Eu0Og7QYNmqHQmEo2qS4FWPlsmSpCaJXpky5Q+ElZbgtTgdAAAA7Cw9NU5v3txHHpdD17y8QtuzS6yOBNiGaZp6eM4GJce6ddeQTlbHARo0WxU9XpdDTodDHX2J6to8UdGoqYhpqnVqHJN8AQAA4Iy1SYvXmzf3lWTo6r+u0K4cv9WRAFuY980BrdqZp18N76zkOLfVcYAGzVZFT1qCV8FwRA7DkMfpkMflUGqcRy1S4qyOBgAAgHrinCYJevPmPgpHTV391xXKzCu1OhJQp5UFI3pi3iZ1a56kH/VubXUcoMGzVdET43aqZUqcHEbFsuqmpJYpjOYBAADA2dXJl6jXb7xI/mBEV7+8QvsLyqyOBNRZf/l4h/YXluvhK7rL6WACZsBqtip6pP+VPbEepzxOByUPAAAAakT3Fsl6/caLVOAP6ZqXVyq7qNzqSECdk5lXqpc+3qEx57fQRe1SrY4DQDYseo4wVDGiBwAAAKgp57VqpP+74SJlF5Xr6pdXKqckYHUkoE55Yv4mGYZ0/8guVkcBcJh9ix6aHgAAANSCXm1S9Lfre2tffpmufXml8v1BqyMBdcLnO3L0/vqDuu0HHdSiUazVcQAcZsuipzwUUTAcVVkorH35pSoPRayOBAAAgHqsT/s0vXxdhnbm+HXtKytVWBqyOhJgqXAkqkfe26hWKbG6eUB7q+MAOIrtip7yUETfZhcrGIkqGI5qf2GZvs0upuwBAABAjerXobFemtxL27JK9JNXV6m4nLIHDdcbK/doS1axfjOqK/OmAnWM7Yqe/fmlyisNyWEYchiGXA6H8kpD2p/PspcAAACoWT/o3FR/uuZCbdhXqOtf/UL+QNjqSECty/cH9eyirerXIU3DuzezOg6A77Bd0ZNdHJDTUTFUMBCOKt8flNNRcTkAAABQ04Z28+kPP+6przILdONrX6gsyMhyNCwzFm1RSSCsaWO6yzBYTh2oa2xX9ATCER0qKpdkyDAk05QOFZUrEOYfWAAAANSOy3s017M/PF8rd+bpltdXM40AGoyN+4v05so9mty3jTr5Eq2OA+A4bFf0eJ0OlYWiCh+eoyfXH1BZKCqv03aHAgAAABsbe0FL/W7iefpkW45ufeNLBcNRqyMBNco0TT08Z4OSY926a0gnq+MAOAHbtSMxHqeipqmK/6RQJKKoaSrGwwRgAAAAqF1XZqTrifE99OHmbP38zS8VilD2oP6a980BrdqZp18N76zkOLfVcQCcgO2KnohpKj0lXjFup5yGoebJcUpPiVfENK2OBgAAgAbo6j6t9cgV3bVwY5bu/NdXClP2oB4qC0b0xLxN6tY8ST/q3drqOAC+h8vqAKcqzuOWv6xiha1I1FRWcZkcptSsUZzKQxGW9gMAAECtu+6StgqGo3p8/iZ5nA49c+X5cjqYpBb1x18+3qH9heV67kc9eW0DdZztih6vy6GQWTEJc3k4oqyicsV5nGoajWpffqlapsRR9gAAAKDW3TygvQLhiJ5ZuFUep0NPTughBx+IUQ/szS/VSx/v0JjzW+iidqlWxwFwErYremSaMmQoappyGIZaNYpTcXlYBaVBtUqNV25JQC1T4qxOCQAAgAbo54M6KhiO6g8fbpfbZejRseey/DRs74n5m2QY0v0ju1gdBUA12K/oMQzFe51yOAyZpinDYahlSqyC4ahKykMMIwQAAICl7hraSYFIVC99/K08TqceHN2Vsge29fmOHM3/5qDuHtpJLRrFWh0HQDXYrujxuhwqDUbkchiVEzCHI6Zi3U6VhiJq6rLd/NIAAACoRwzD0H0juigYjupvn+2Ux+XQvSM6U/bAdsKRqB55b6NapcTqlgHtrY4DoJpsV/TEe13KLSmXy+GQaUpl5WEVRgJqnRYvQ1JagtfqiAAAAGjgDMPQQ6O7KRSJ6sWPd8jjcujuoZ2sjgWckjdX7dGWrGK9eO2FzIMK2Ijtih5/IKwuzZPlcuxX1JQ8bofcbodKAxFltE3jBxAAAADqBMMwNP2KcxUKm/rDkm3yuhy67bIOVscCqiXfH9SMhVt1yTlpGt69mdVxAJwC2xU9gXBUjRO8apzolSOrWG0bJ8jjNOR1OdUozmN1PAAAAKCSw2HoiQk9FIxE9fSCLfI4HbqZU2BgAzMWbVFJIKxpY7pz2iFgM7Yrerwuh0IRUx5nxVw8bdLiFQxHxRzMAAAAqIucDkNPTzpPwXBUj8/fJI/LoesuaWt1LOCENu4v0psr9+gnF7dV52aJVscBcIpsV/SkJXi1L79UMqRI1FQwHFUwHGFJdQAAANRZLqdDz/3oAoUiUU17b4PcToeu7tPa6ljAMUzT1CNzNig51q27hjCvFGBHtluiKsbtVMuUODkNQ5GoKYchtUyJY24eAAAA1Glup0N/vLqnLuvcRL+e9Y3eXrPX6kjAMeZ9c0Ard+bpl8M6KznObXUcAKfBdkWPVFH2NIpzyxQlDwAAAOzD63LqL9f2Uv8OjTX17a81+6t9VkcCKpUFI3pi3iZ1bZ6kH1/EiDPArmxZ9EgVE9uFo6bVMQAAAIBTEuN2aubkDF3ULlV3//trvf/NAasjAZKkFz/eof2F5Xrkiu5yMgkqYFu2LXpcDkNRih4AAADYUKzHqVeu662e6Y30i7fWatHGLKsjoYHbm1+qFz/eodHnNddF7VKtjgPgDNi26HEajOgBAACAfcV7XXp1Sm91b5ms2974Uh9tybY6EhqwJ+ZvkmFID1ze1eooAM6QfYseR0V0RvUAAADArhJj3Pr7lIvU0Zegn76+Rp9tz7E6Ehqgz3fkaP43B3XrDzqoRaNYq+MAOEO2LHoKSoM6WFQmSVq1K0cFpUGLEwEAAACnJznOrddv7KN2jeN102urtWpnntWR0ICEI1E98t5GtUqJ1S0D2lsdB8BZYLuip6A0qLW78ypH8uzJKdPijQd0sLDM4mQAAADA6UmN9+gfN/VRi0YxmvLqKq3ZnW91JDQQb67aoy1ZxfrNqK6sZgzUE7YrenYcKlbUlArLQpKk7OIyHSoO6P31BxjZAwAAANtqnODVmzf3VZNEr67/2yqt21tgdSTUc/n+oGYs3KpLzknT8O7NrI4D4CyxXdGTVxLU/oIyBUJRSdKXuwu0eX+RDuaXaeP+QpWHIhYnBAAAAE6PLylGb97cV43i3Zr8yipt2F9odSTUYzMWbVFJIKxpY7rLMFhOHagvbFf0RExTOSUBfX34Nxz7CssUkbQzt0Q5JQHllgQszQcAAACciRaNYvXmTX0V73Fq8iurtOVgsdWRUA9t3F+kN1fu0eS+bdS5WaLVcQCcRbYrepolx+rbQ8WVI3pyioMyTVNp8R7ty/crEI5anBAAAAA4M+mpcXrz5r5yOQxd8/JK7ThUYnUk1COmaeqRORuUHOvWXUM6WR0HwFlmu6LH6zAUjphyHp4nLBCJqqgsJKchBSOmvC7bHRIAAABwjLaN4/XmzX0lmbr6ryu0O9dvdSTUE/O/OaiVO/P0y2GdlRzntjoOgLPMdq1IYXlYaYkeJcZU/EAqCUSUXRjQ7pxyeRxOpSV4LU4IAAAAnB0dmibojZv6KhiO6uq/rtTe/FKrI8HmyoIRPT5vo7o2T9KPL2ptdRwANcB2RU8gHFGb1HilxroqL8v2B5VbWqaEGJYDBAAAQP3SuVmiXr+xj4rLQ/rxX1foQGGZ1ZFgYy9+vEP7C8v18JhucjqYgBmoj2xX9CTEuHSoJKDkeE/lZYZhqE3jBJUHo0zGDAAAgHrn3JbJev3GPirwh3T1X1cqu6jc6kiwob35pXrx4x0afV5z9WmfZnUcADXEdkVPy0Zx2rC/UHtyiiovKyoPqywY1X4mYwYAAEA9dX56I/3fDb2VVVSuq19eqRx+wYlT9MT8TTIM6YHLu1odBUANsl3RszvXr6LScOWqW5IUiZram1uivYUBBYJhC9MBAAAANadXm1T97fre2ptfqmtfXql8f9DqSLCJz3fkaP43B3XrDzqoRaNYq+MAqEG2K3pWfpujxglexXj+Nx+Py1ExqkcylV3MbzYAAABQf/Vtn6aXf9Jb3+b4NflvK1VYFrI6Euq4cCSq6XM2qlVKrG4Z0N7qOABqmO2KnuLysNwuQ+XBo0f0SJFoRHEuQ5sOFGlffqnKQxELUwIAAAA1p3/Hxnrp2l7acrBY1/1tlYrLKXtwYm+u2qPNB4v168u7KsbNAjZAfWe7oseX7NXBgoBKjypyTEnlIelgUUDBcESlwQhlDwAAAOq1y7o01Z+uvlDr9xXqhv/7Qv4AUxjgWPn+oGYs3KpLzknTiHObWR0HQC2wXdHTo2WKSoIheRzSkdXUzcP/78krVVimth4sUlkowgpcAAAAqNeGdW+m53/UU2t25+um11arLMgvOlHVs4u2qiQQ1rQx3WUYLKcONAS2K3qaJMaoWWKM3A7JYUqGKv6PmlI0asoMS2HT1K5DJYfn7QGsVR6qGGH27aESRpoBAICzbtR5zfX7qy7Qip25uuX11bzXQKWN+4v0xsrdurZPa3Vulmh1HAC1xGV1gNOREONUYnysDEdIQX9YEVMyo1J5NKJ1e/NUHgrJcDqUWxqU1+VQi0axlp2LWn54ZFEgHJXX5VBagpfzYhuQIyWPx+VUnMepUMTUvvxStUyJ43VQy/heBADUZ2MvaKlAOKqpb6/TbW98qb9c20sel+1+p4uzyDRNPTJng5Jj3bpraCer4wCoRfYrekxTLVJi9fW+ApUHIxUlj6SApEC59PmOAm09WKy2jeNlRk2t9uSqQ1miujVPrvUPdUc+5JumVBIIqTQQ0Z5cv7q2SFajOE+N3zcfaq2XWxKQx+WsfKPlcRmVl7dMiTvl/Z3p89pQXxcUbgCAhuCHGekKRaL69bvrdftba/XHq3vK7aTsaajmf3NQK3fm6bFx59b4Zw8AdYvtip5AOCpfUpxSYtzaXHzssNSQpAP+iMrCRWrWKFZN/UEt356jndklatEoVrEep8pDURWXhxQxTTVLilWTRO8xH3jLQxHtzy9VdnFApmnKlxxbOTLo+647ctvckoAy80oVjpqKmqYSY9xKjHWrLBjRxv2FurBNao19wDyVD7Xf/eAf73XJHwh/bxFQF8qCupChOnkC4ajiPFVzuZ2GSk/j/PnTKSuOzhUIhpWZX6ayUEQuw1CjeI8KS4Nq3zTxrD12p/q8nMr2Z/Kcn0nhVtdea0ery9kAVFi2bJkef/xxRaNRXXnllbrllluqXL9v3z498MADysvLU6NGjfT000+rWbP/TZZaUlKiyy+/XEOGDNFDDz1U5bY/+9nPtHfvXs2dO7dWjgX2cE2fNgqGo3pkzkbd/e+v9dxVF8jpYF6WhqYsGNET8zepa/Mk/fii1lbHAVDLbFf0lIbCSopxyzS/f7uCgLRoQ5Y2HyiSx+lQj/QUGZLWZhaoLByRxzCUlujVoaKAurVM0p5cv8Kmqcw8vwIhU+FoRL7DJVBRWVh783O1OzFGXZsn6UBBqTYeKNahonJFZCrR41KXFslqlRKnQCii/NKgmiTGyDAkf3lYJcGQ/MGwQmFTgXDFn163Q91bNDqmHDrygc3pMLSvoFR5JcEqhdTRRYxMUzIMBUIRlYbCivO4lRTjUiAcrfKhNmpGlVcaVHZRuVqnxVd+GDxSHJSHItpfUKac4oBKysPq1DxRwXBUOcUBBcNRdWmRpFYpcUpL8EpSZdngMKSduX4t25otj9OhxolepafGn/RUuSPHWlQWqpK7uh9Sz2R0xtGP85HHT9L3fkguKA1qx6FilZSHlRDj0jlNKoqRo/dTFo4qKcatOI9TJeVhrd2dp0bxXhX6A4pIcjsd8jgdSon3yGEY8p7GUOpTLSuOfpwchvTNviLl+iteA16nUwcKy7SvsFSHigPq4Es845LgVJ+XUy0kv7vtt9nFivFU/Ag7WclxuoXbmZSmNV26fDfb0a+7U/l+AlBzIpGIpk+frldffVU+n0+TJk3SoEGD1KFDh8ptnnrqKY0bN07jx4/X8uXLNWPGDD399NOV1z/33HPq3bv3MfteuHCh4uPja+U4YD9T+rVTMBzVk+9vlttp6JlJ58tB2dOgvPjxDu0rKNOzPzyfog9ogGxX9MR53CoPhVUcDJ1025KwtC27TLFuKac0oBXf5ijB61aMy6EYl0N7C8pVHsrV0s1ZapYUo9ySgJLjvSooKVfR4YLG5XDKaUgxLodapsZr7a4cRSXtLwxK0aiiMlQWDOvL3Xnq0aqRTElZheUqD0cV63ZIMhU1DTkkud0Oed1ORYJh7ThUrA++3qfUpFi1To1TvMellilxSoxxaWdOiT7alC23q+JDfHZRSFnFpXI7HPK6XWrTOE7JMS5FzIpcLpdDoVBUbqdD3Vo2kiFTnZolqTxkKquoTPvyy+VySKWBsPbllamgPKg2jeOV6HUrKlOb9hfJaRgKRiMqKg/qXyvy1DTZq9JgRG6HU5l5Jerdvom8Lodi3E4lxLiUUxLQ9uwS7c31K6c0JBlRpcZ61aZxmdo1iZcvMVaSjhkldKQU8TodKiwLyWEYKiwNKBKNak+uv8qH1ILSoL7KzFdBaUiN4tzq0ixJbqdDmXmlikSjMmVoX36ZsovKFAxHFA5HFRfjVqzHqY6+RJ2fnqJGcZ4qxdKREs7tNLS3oFySqSYJXuX6A9qWVaxGcW6Vh6IKhCNKiHEpwePSN3sLVR6OKL80oOLysKLR/ergS1J6SpyipqndeX45TKl14wSZMpVTHFQgHNHXmfk6VBRQWSiqri0SlBzr1basIjVOjFHP1inf+9o9XiGVmedXUoxbqUd9gI9Eo9pXWH7ccuHoYuhgYUDFgZBM09SenFKlJHjkD4TldTpUEgirLBipLAm8TqNKAfbdUV7HG/UlSRv2F6g8FFW8x6WU+IrhwQeKyrVxX6EaJ3qPGfl2suLq6McgvzSgpBjPd8rLkLzBiNJT46oUMOWhyDHFnNflUChiVt6HJIUipmRWFEYnGp23v6BMef6gTKmyqPO4Kkq+tATvCcu+ExVC1R1Bd7JRg999/MpDEeX6g3I6HCoPhuV2GFVKn6PvJxAMq7A8XPka/25xefRzeqJS9PteE9UtUO2AEVM4U+vWrVObNm2Unp4uSRo1apSWLFlSpejZsWOH7r//fklS3759ddttt1Vet379euXm5urSSy/V+vXrKy/3+/169dVX9eijj+rOO++snYOB7fx04DkKhqOasWirPE6Hnhjfg7KngdibX6oXP96h0ec1V5/2aVbHAWAB2xU9STEuxXqdCldzQa2wpOKQ5C8MK6qwpHJJkldScqxUEpAC0f9tH++u+IxSGqw4DUz639Jknt2FinFLXrdDpkyVB0wFwlJUFfMErd6VoxivW05TKg2HVV5mKhiVYj1SrNepxFiXSgNROQ2HkuOcahwfqy0HivVRNKoEj1uxMS4Vl4a0JatQwVBEpYEjaatas6tAXofkdUnl0YqJqBNiJIcccjgNNU6IUaLXJVOG/IGwTKPi9DGPw1Cj+FiVlAdVXBZU2DRVGjQV7zUU63bL63aqLBhScTCk9QcMJcS65HU4VBaK6svdBWqaFKOoTMU4DIUNaV9+mXJLymVGTSUneNUk3qvtB4vl9TjUKNathFi3/IGwcooD8rqckiH5gxFFwhF5PC55HIbcLodKgxFFIqZSEjxyGVKcxyV/MKQD+eUqKgvJHwopHJVi3S4NP9enWJdL27NKtCO7RAWBoMKKKBSIyulyqnVKnNqkxmt3TolW7MhRnNel3YeKVR425TAMNU70qFlynFLiPYpxOeUPhLVsc7YCkYj8gaiKy4Nq2ShWSXFu7c0v04a9BZIhlYejikaj8jidCpsRzfpyj7wep2LdbjWJ98rXKEZf7s5VxJRcTof8wYgCwag6N09QSTCst1ZkKmRGZBhSaqxHPVqlqIMvUVlF5dqT61cwHFWM2ymHU4pGpLJgWHExLjkkBcOmkmPdSopzqjRoqqQsqJR4r1Li3couCioSNRXvdaioLKh9hQHFOJ1Ki/eocbJHbVMTFOd1a/XuXK3YlqNgJCKn06GmCbFKjHWqSaJXBWVBZRWVqzQYVmkgpIhpKMbjVIvkGEVMqdAfVPumCWoU59G+3DKt2Z2riKRQNCq3w1B5KCpDprwul9JTY+R1u1VcHlQoYsrlkJxOh/LLg/p400EFIlJirEsup6EDBWUqKQ/LlKGkWJfaNo5XekqcCstCCkWi2pFdrLJIVA4ZMmSoa/Mk9TknTcmxHmXm+pWZ51dxIKLs4nIlx7pVWh7R0i0HVVQWUZzHIa/TodJgVEs3HVT3Fo1UUBZSSSCsSCQqh6Ni1JwhU2WhiuLR7XIoM79UHX1J6to8SeWhiFZ+m6uC0oAOFZWrNFRxvJ2bJyne7VIgElVuSbkC4YhKghGVByNyOSS306XkeLfap8UrM79UMS6nAuGIAuGocksCkkyFo1FFTam0PFL52EYiprZnFSvG69CGvUXafrBIUUNyGoYMU4qNcalJolcxrooiNGpG1aZxnEzT0M5DfuWWBJTvL1d2SVAuw5DT4ZDLMJUQ61YwYqp7y2TFuZ3aklUspyG1TouXYRj6ZEu2DKcUiUhOQ0pJ8GpPvl+BYFSmpPySoA4UlcntMNS1RZLivC7tyvYrNdGrWLdDZcGoDhSWqU3jeHmc0pd7CrQ/z6+oKXlcLiXFutShaaISYtwKhioeB6/HqdR4t85pknhMGVsaCsvpcCgSjSrO4z6meDy6bDl6tJ3b5VCcx6nyYLRKOSbptEYQns4ca8cb/ffdbQtKg9p4oFAH8svkcTnUwZeodo0TKgvKo0dxJsW4jnvsxyugjhwnpVTdkpWVVeU0LJ/Pp3Xr1lXZpkuXLlq4cKGuu+46LVq0SH6/X/n5+UpOTtZTTz2lp59+Wp9//nmV2zz//PO64YYbFBMTUyvHAfv6xeCOCkai+uOH2+V2OjR9LMtrNwRPzt8sw5AeuLyr1VEAWMR2RU9agleG6VBcjFMqrf48J9HvfB2QlF127HZFxxkodOS25aZUHpQU/O7eKuSUSSo7dgeBoFQQjOjA0XMKFUhS6cliH1dIUigqlQT/d1lp6f+S7i/2f8+tT3Rd4NiLCv53BzGGtHFf8XGLJ0naV1yq0z2eWElHP5Nep3Sc6ZckBbTh4Lffs6eIMvOD+mpXgWRUvDyikgxVlHXf92pxHd7uyBmBkaP+XtVRz29ZRFJE23PKpd2Fx9166dacKq89h6SoyrVid1HlZR5V5Dy6u3Qdtb1x+P+oJK9b8roNuRymykNSNCp5XBVlX37gf69V9+Ht3Tp+WSiVHM5Scf8epxQ1K15XpiTn4ft0GhX36XQacnsMlQeiMkzJ6awotIpKowpHJYdZcYOIWXFbt/d/l3k9Unmg4nvn6CxuVTzX4YjkdFRkCEuKhKQy/e/xT3RLMU5p9uqK64+ctulySG6H5HBW3CbikMLhiufuyGMZ45QSXFI4WnF5ICQ5jIr7cjmk8rAUiUoxbqlV41ilxMYoGt2rkvKg9heWqTRgynBIHkfFY2MYUsw6KTHOLZfLpZKygPJLoio3K8rjRomGUmNjFON26iPT/N9rzjSVUxpQIBiRYRgVz6nDVHKsR4Ycine75Y+ElVNYJv+RB+nwE+R0Sg6HFOuSTMOQYZiHRwi6FOt2q21agrKLy5VXUq5INCq3yykZZuXj6nIachimlmzaL38gIn95RMFwxeMY55Hi41yKcTuVGhcrhyE5ZCgYjSoYCim7uFwl5VG5HVJqolcuZ0VZa8ioeCycTjkMh9wuQ26HoZApxbqdOlRcptyioMrCFa8Hj1tKTXAr0euR2+lQVGblwJ/2TRPlMSSny6GS8qCyC4PKLQ0qzuNQnMet5DiPnEZF3vJQRa3eJq1ipNSOQyXKLwkqEA6rLBRRxDTUNMErt8cpM2KqPBRVJBqVIUMRReVyOGUYptwOh8JRU4mxbrVOjVXLlDilxccoYkaV5w9od45f+aUheR2Sw+moPP0xGDEVXL1HLZJj1DQxRk0SYtQqNU5ep6GdOaX6ck+uHIYU53HK5XLq023Z8iXFKd8fUGkgrIRYtwr9QZWFozKjURUFInrriz1yquJDV3KMU4bDUHnYrCgOXQ7FexxqluBVUSBSOVo0Odalzs0SlJYYo5yigAKhqDxuh4rLQyoPR9QozqPzW6WofZMEHSoOqLA0qKQ4d7VOrUXtmzp1qh599FG9++67ysjIkM/nk9Pp1JtvvqkBAwZUKYokadOmTdqzZ48eeOAB7d2716LUsJO7h3ZSMBzVS8u+lcfl0G9GdaXsqcc+35Gjed8c0F1DOlX+wgNAw2O7oifG7VSLRjFKjYvRrrzgyW+As6L8JHMinYnv9m2nMU9xJVNSSfTYy062y2oOEDtl360Ej1cRHu9VfKI8wZBUHPrOk3GccvLIRd933EeyBHXsY155/6bkDx7+S9l3XwTHeaCPCJzg79/JGDpyv1HJf/z+VMWhiv+/KxhV1YbsOLcvj1T8/92c5d95gAMhqfBAmY59NR7O9p08h8pD+u4DXy7pYLGpg8XH2cd3AxyWVdnWHqeOO3I8FX3i4cfg6Ac5LCmsrYe+e3/V/wYKBKX8YPjwvk7wRB2Wn3u8609+Cm1IUnlIKso/9jGTpK/2lVQn6nfknvCaHbknqqNPn0OVvVuVMtXrqHgdHimGDR3/0fcevn1Ax/8ZcDoWbs457uVxOvy9pd1V7r9xkkudmiXpigtaamSPlpQ9tcTn8+ngwYOVX2dlZcnn8x2zzQsvvCCp4pSshQsXKikpSWvXrtWaNWv01ltvye/3KxQKKS4uTi1atND69es1aNAghcNh5eXlafLkyXr99ddr9dhgH4Zh6L6RXRQIR/XKpzvlcTk0dXhnyp56KByJavqcjWrZKFY/Hdje6jgALHRGRc/JVpKoKU2TYtS9ZaK+2lt01t40AwBwPEcXPEccGVlZHd9foZ1dxxtXGZC0ryisfUV52rg3XzEup0ae17IWUzVcPXr00K5du5SZmSmfz6d58+ZpxowZVbY5stqWw+HQzJkzNXHiREmqst0777yj9evX61e/+pUk6eqrr5Yk7d27Vz/72c8oeXBShmFo2phuCkai+stHO+R1OXTnkE5Wx8JZ9taqPdp8sFh/ueZCCn2ggTv1pX8OO7KSxMsvv6x58+Zp7ty52r59+9nMdkJJsW51a56iBHet3B0AAPVCVqmpZxduUnnoDIZOotpcLpceeugh3XTTTbr88ss1cuRIdezYUc8//7yWLFkiSVq1apVGjBih4cOHKycnR//v//0/i1OjvjIMQ4+NPVdX9mql5xZv058/qp337agd+f6gnlm4VRe3T9OIc5ud/AYA6rXTHtFTnZUkakpaglfNk2PUpnGCvjlwOsP+AQBomLblBCpXt0PNGzhwoAYOHFjlsjvuuKPy7yNGjNCIESO+dx8TJkzQhAkTjrm8VatWmjt37tkJigbB4TD024nnKRSJ6ncfbFEkYqpfx8ZWx8JZ8MaKPSoJhDXtim6clgfg9Iue6qwkUVNi3E5d0CZFl3ZMU2Z+iQrO/pQMAADUW4EwJz4DDZXTYeiZK89XKGJqxqKtmrFoq9WRcJZcf0lbdWmWZHUMAHWA7SZjPqJRnEc/6XeOsoqDWvFtjg4UhpivBwCAk2jkqViuHUDD5XI69PyPLtC1fdsoEOZUzvrA43KoT7s0q2MAqCNOu+ipzkoSNa1Zcqx+MbiTWiTHaMO+Im3OytfBoiiFDwAAJzCpd2ulJXitjgHAYi6nQxefQzEAAPXRaRc91VlJoja0bZygGwd00MYDhdqRVaytBwq1O69cef5y+QMhmZJC0YgkQzFut1okx6l1I68y88u1I6dQBSVRlUdPvCCxV1KjeCkUkvKDFUvoOiTFqGIlk6Nv55IU65Aczoplds2oVBI5lcWOjy/RJTVOcis5JkaSqZzSMuUWRI63EPQZO3JGr0MVx0ppdmIxkpLipHBIKgpVPM9uSfEeSWbFctjfXSbdqYrH9uSLUgPA2eWRdOOANrpjaFdWYwEAAKjHTrvoOXoliUgkookTJ6pjx45nM1u1NYrz6JJzmuiSc5pYcv+oGeWhiHJLAgqEo/K6HEpL8FZ+OPm+6+qimsprt8cBNeNsvQ7s9HqqS1mPZCkqC6k0FFacx62kGFedfvwAAABQf53RHD3HW0kCOFti3M4TrgrzfdfVRTWV126PA2rG2Xod2On1VJeyHsnSMsXqJAAAAEDFWSQAAAAAAACoByh6AAAAAAAA6gmKHgAAAAAAgHqCogcAAAAAAKCeoOgBAAAAAACoJyh6AAAAAAAA6gmKHgAAAAAAgHqCogcAAAAAAKCeoOgBAAAAAACoJyh6AAAAAAAA6gmKHgAAAAAAgHqCogcAAAAAAKCeoOgBAAAAAACoJyh6AAAAAAAA6glXTd/BmjVravouAAAA8B28BwMAoGEyTNM0rQ4BAAAAAACAM8epWwAAAAAAAPUERQ8AAAAAAEA9QdEDAAAAAABQT9iy6Fm2bJmGDx+uoUOHaubMmVbHsYUDBw5o8uTJuvzyyzVq1Ci99tprkqSCggJNmTJFw4YN05QpU1RYWChJMk1Tjz32mIYOHaoxY8Zow4YNlft69913NWzYMA0bNkzvvvuuJcdTV0UiEY0bN04//elPJUmZmZm68sorNXToUN15550KBoOSpGAwqDvvvFNDhw7VlVdeqb1791bu46WXXtLQoUM1fPhwffLJJ5YcR11TVFSk22+/XSNGjNDIkSO1du1aXrtn0f/93/9p1KhRGj16tO6++24FAgFeu2fg/vvv18UXX6zRo0dXXnY2X6/r16/XmDFjNHToUD322GNiqj3UlJO93/q+nwd2cbJjfOedd9S3b1+NHTtWY8eO1X/+8x8LUp654/1cOtr3/Syyi5Md48qVK9WrV6/K5/KFF16o5YRnx4ne0x/N7s9ndY6xPjyfgUBAkyZN0hVXXKFRo0bpD3/4wzHb2P3nbHWOsb78nJWO/Sx4tFp/Lk2bCYfD5uDBg809e/aYgUDAHDNmjLlt2zarY9V5WVlZ5vr1603TNM3i4mJz2LBh5rZt28ynnnrKfOmll0zTNM2XXnrJ/N3vfmeapml+9NFH5o033mhGo1Fz7dq15qRJk0zTNM38/Hxz0KBBZn5+vllQUGAOGjTILCgosOag6qC//e1v5t13323ecsstpmma5u23327OnTvXNE3TfPDBB8033njDNE3T/Mc//mE++OCDpmma5ty5c8077rjDNE3T3LZtmzlmzBgzEAiYe/bsMQcPHmyGw+HaP5A6ZurUqea///1v0zRNMxAImIWFhbx2z5KDBw+al112mVlWVmaaZsVr9r///S+v3TOwatUqc/369eaoUaMqLzubr9eJEyeaa9euNaPRqHnjjTeaH330US0fIRqC6rzfOtHPA7uozjH+97//NR955BGLEp49x/u5dLQT/Syyk5Md44oVKyrfn9nZid7TH83uz2d1jrE+PJ/RaNQsKSkxTdM0g8GgOWnSJHPt2rVVtrH7z9nqHGN9+Tlrmsd+FjxabT+XthvRs27dOrVp00bp6enyeDwaNWqUlixZYnWsOq9p06bq3r27JCkhIUHt27dXVlaWlixZonHjxkmSxo0bp8WLF0tS5eWGYeiCCy5QUVGRsrOz9emnn6pfv35q1KiRkpOT1a9fvwb9m/ujHTx4UB999JEmTZokqeK3KStWrNDw4cMlSePHj698rX744YcaP368JGn48OFavny5TNPUkiVLNGrUKHk8HqWnp6tNmzZat26dNQdURxQXF+uLL76ofFw9Ho+SkpJ47Z5FkUhE5eXlCofDKi8vV5MmTXjtnoHevXsrOTm5ymVn6/WanZ2tkpISXXDBBTIMQ+PGjePfQNSI6rzfOtHPA7toSO8pj/dz6Wgn+llkJyc7xvriRO/pj2b357M6x1gfGIah+Ph4SVI4HFY4HJZhGFW2sfvP2eocY33x3c+C31Xbz6Xtip6srCw1a9as8mufz1cvv/Fr0t69e7Vp0yadf/75ys3NVdOmTSVJTZo0UW5urqRjH+dmzZopKyuLx/97PPHEE7rnnnvkcFR8W+Xn5yspKUkul0vS/x5DqeLxbd68uSTJ5XIpMTFR+fn5PL7HsXfvXqWmpur+++/XuHHj9Otf/1qlpaW8ds8Sn8+nG264QZdddpn69++vhIQEde/endfuWXa2Xq8n2h4426rzPX2inwd2Ud2fWwsXLtT/b+9+XqL6wjiOv2UsEqbUiZxBaqMZSFZLZZApRzQLTKIkXLQIpEXSUJPSL7JVuSmzv6FFmxZj4WIW3rBEI0gmFy4KQhDKDIwxQtEZ73cR+tWaycEmr/f2ee0cL8N57nN8OOfxzpnGxkZCoRCfPn3ayCFumH+ltsRiMU6cOEFrayvv37+3ejh/bOWafiUn5TNdjOCMfCaTSZqamvD7/fj9/pS5tHOdhbVjBGfU2Z/3gj/b6FzartEjf+b79++EQiFu3LiB2+1e9bucnBzHdlj/tufPn+PxeKioqLB6KI6TSCQYGxujpaWFSCRCXl7eL+coaO6uXzwep7+/n/7+fl6+fMns7KyedPrLNF9F7KOmpgbDMHj27Bl+v5+rV69aPSRZp/3792MYBk+fPuXs2bO0tbVZPaQ/8rs1vVP8Lkan5NPlctHb28vAwACjo6O8e/fO6iFl3VoxOqHObsa9oO0aPV6vl8nJyeWfP3/+jNfrtXBE9rGwsEAoFKKxsZH6+noAdu7cufwo59TUFB6PB/j1Pk9OTuL1enX/0xgZGcEwDILBIOFwmFevXnHnzh1mZmZIJBLA//cQftzfpW51IpHg27dvFBYW6v6m4PP58Pl8y93/hoYGxsbGNHezZGhoiN27d+PxeNiyZQv19fWMjIxo7mZZtuZruutFsi2Tv+l09cAuMomxsLCQrVu3AtDc3Gy7Q20z9S/UFrfbvfwRksOHD5NIJJienrZ4VOuTak2/khPyuVaMTsonwI4dO6isrPzln212r7MrpYvRCXU21V6wvb191TUbnUvbNXoOHDjA+Pg4ExMTzM/P09fXRzAYtHpYm55pmty8eZOSkhLOnTu3/HowGCQSiQAQiUSora1d9bppmsRiMbZv305RURHV1dUMDg4Sj8eJx+MMDg5SXV1tRUibypUrV3jx4gWGYdDd3U1VVRX379+nsrKSaDQK/PgGnaW5GgwGl79FJxqNUlVVRU5ODsFgkL6+Pubn55mYmGB8fJyDBw9aFtdmsGvXLnw+Hx8+fABgeHiY0tJSzd0sKS4u5u3bt8zOzmKaJsPDw+zdu1dzN8uyNV+Liopwu93EYjFM01z1XiLZlMl6K109sItMYlx5rolhGJSWlm70MDdEulrkJF++fFk+D2N0dJTFxUVbbpjTrelXsns+M4nRCfmcnp5mZmYGgLm5OYaGhigpKVl1jd3rbCYxOqHOptoL3rt3b9U1G53L3L/2zn9Jbm4unZ2dtLa2kkwmOXXqFGVlZVYPa9N78+YNvb297Nu3j6amJgDC4TDnz5/n0qVLPHnyhOLiYnp6eoAfnfGBgQHq6urIy8vj7t27ABQUFHDhwoXlQ6ba2tooKCiwIiRb6Ojo4PLly/T09FBeXk5zczMAp0+fpqOjg7q6OvLz83nw4AEAZWVlHDt2jOPHj+Nyuejs7MTlclkZwqZw69Yt2tvbWVhYYM+ePXR1dbG4uKi5mwWHDh3i6NGjnDx5ktzcXMrLyzlz5gxHjhzR3F2ncDjM69ev+fr1K4FAgIsXL2a11t6+fZvr168zNzdHIBAgEAhYEaY4XLr11sOHD6moqKC2tjZtPbCLTGJ89OgRhmHgcrnIz8+nq6vL6mGvS6q6tPTUZktLS9paZCdrxRiNRnn8+DEul4tt27bR3d1tqw3zknRr+o8fPwLOyGcmMTohn1NTU1y7do1kMolpmjQ0NFBTU+OoOptJjE6ps6lYmcsc007HdouIiIiIiIiISFq2++iWiIiIiIiIiIikpkaPiIiIiIiIiIhDqNEjIiIiIiIiIuIQavSIiIiIiIiIiDiEGj0iIiIiIiIiIg6hRo+IiIiIiIiIiEOo0SMiIiIiIiIi4hD/AQiShDiJvkCQAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Val accuracies: [0.9472477064220184, 0.9438073394495413, 0.9438073394495413, 0.9495412844036697, 0.9552752293577982]\n" ] } ], "source": [ "print(f'Iteration: {progress_bar.n}')\n", "plt.figure(figsize=[20, 8])\n", "plt.subplot(1,2,1)\n", "plt.title('Train loss over time', fontsize=12); plt.grid();\n", "plt.plot(moving_average(loss_history, span=10))\n", "plt.scatter(range(len(loss_history)), loss_history, alpha=0.1)\n", "\n", "plt.subplot(1,2,2)\n", "plt.title('Val accuracy', fontsize=12); plt.grid();\n", "plt.plot(accuracy_history)\n", "plt.show()\n", "print('Val accuracies: ', accuracy_history)" ] }, { "cell_type": "code", "execution_count": 16, "id": "06daf317", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.10097131777508184" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sum(loss_history[-1000:]) / 1000" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.12" } }, "nbformat": 4, "nbformat_minor": 5 }