From 8df8c8d90d027ea098ce8edc98c87effeebf3077 Mon Sep 17 00:00:00 2001 From: Janaki Date: Thu, 6 Feb 2025 09:11:20 -0800 Subject: [PATCH 1/9] DAPT with NeMo 2.0 --- .../domain-adaptive-pretraining/.gitignore | 8 + .../domain-adaptive-pretraining/README.md | 35 + .../code/__init__.py | 71 + .../code/custom_tokenization.ipynb | 1920 +++++++++++++++++ .../domain_adaptive_pretraining_nemo1.0.ipynb | 489 +++++ .../domain_adaptive_pretraining_nemo2.0.ipynb | 553 +++++ .../code/extend_tokenizer_utils.py | 275 +++ .../code/get_high_freq_tokens.py | 114 + .../code/imgs/embedding_table.png | Bin 0 -> 69341 bytes .../code/imgs/tokenization_diagram.png | Bin 0 -> 113644 bytes .../code/tokenization_helper.py | 399 ++++ .../domain-adaptive-pretraining/code/util.py | 65 + 12 files changed, 3929 insertions(+) create mode 100644 tutorials/llm/llama/domain-adaptive-pretraining/.gitignore create mode 100755 tutorials/llm/llama/domain-adaptive-pretraining/README.md create mode 100644 tutorials/llm/llama/domain-adaptive-pretraining/code/__init__.py create mode 100644 tutorials/llm/llama/domain-adaptive-pretraining/code/custom_tokenization.ipynb create mode 100644 tutorials/llm/llama/domain-adaptive-pretraining/code/domain_adaptive_pretraining_nemo1.0.ipynb create mode 100644 tutorials/llm/llama/domain-adaptive-pretraining/code/domain_adaptive_pretraining_nemo2.0.ipynb create mode 100755 tutorials/llm/llama/domain-adaptive-pretraining/code/extend_tokenizer_utils.py create mode 100755 tutorials/llm/llama/domain-adaptive-pretraining/code/get_high_freq_tokens.py create mode 100644 tutorials/llm/llama/domain-adaptive-pretraining/code/imgs/embedding_table.png create mode 100755 tutorials/llm/llama/domain-adaptive-pretraining/code/imgs/tokenization_diagram.png create mode 100755 tutorials/llm/llama/domain-adaptive-pretraining/code/tokenization_helper.py create mode 100755 tutorials/llm/llama/domain-adaptive-pretraining/code/util.py diff --git a/tutorials/llm/llama/domain-adaptive-pretraining/.gitignore b/tutorials/llm/llama/domain-adaptive-pretraining/.gitignore new file mode 100644 index 000000000000..ca5fecf18b75 --- /dev/null +++ b/tutorials/llm/llama/domain-adaptive-pretraining/.gitignore @@ -0,0 +1,8 @@ +/code/general_data/* +/code/data/* +/code/models/* +/code/nemo_experiments/ +./preprocessed_data_text_document.bin +./preprocessed_data_text_document.idx +./llama2_7b.py +./test_convert.py \ No newline at end of file diff --git a/tutorials/llm/llama/domain-adaptive-pretraining/README.md b/tutorials/llm/llama/domain-adaptive-pretraining/README.md new file mode 100755 index 000000000000..d2ce82a537d0 --- /dev/null +++ b/tutorials/llm/llama/domain-adaptive-pretraining/README.md @@ -0,0 +1,35 @@ +# ChipNeMo - Custom tokenization + Domain Adaptive Pre-training on Llama 2 70b with NeMo Framework + +[ChipNeMo](https://arxiv.org/pdf/2311.00176) is a chip design domain-adapted Large Language Model (LLM). Instead of directly deploying off-the-shelf commercial or open-source LLMs, the paper adopts the following domain adaptation techniques: domain-adaptive tokenization, domain-adaptive continued pre-training, model alignment with domain-specific instructions, and domain-adapted retrieval models. Specifically, Llama 2 foundation models are continually pre-trained with more than 20 billion tokens on domain-specific chip design data, including code and documents. They are then fine-tuned with instruction datasets from design data as well as external sources. Evaluations on the resultant domain-adapted ChipNeMo model demonstrate that domain-adaptive pre-training of language models can lead to superior performance in domain-related downstream tasks compared to their base Llama 2 counterparts, without degradations in generic capabilities. + +Here, we share a tutorial with best practices on custom tokenization and DAPT (Domain-Adaptive Pre-Training) for a ChipNeMo-like code generation use case. + +## Requirements + +### Software Requirements +* Access to latest NeMo Framework NGC Containers +* This playbook has been tested on: nvcr.io/nvidia/nemo:24.07. It is expected to work similarly on other environments. + +### Hardware Requirements +* This playbook can run on CPUs or GPUs. For GPUs, this playbook has been tested on minimum 2xA100 80G + +### Data Curation + +* In this tutorial, we will leverage chip domain/hardware datasets from open-source GitHub repositories, wiki URLs, and academic papers. Therefore, as a pre-requisite the user should curate the domain specific and general purpose data using the NeMo Curator and place them in the directories mentioned below. + +* `./code/data` should contain curated data from chip domain after processing with NeMo Curator. Playbook for DAPT data curation can be found [here](https://github.com/NVIDIA/NeMo-Curator/tree/main/tutorials/dapt-curation) + +* `./code/general_data` should contain open-source general purpose data that llama-2 was trained on. This data will help idenitfy token/vocabulary differences between general purpose and domain-specific datasets. Data can be downloaded from [Wikepedia](https://huggingface.co/datasets/legacy-datasets/wikipedia), [CommonCrawl](https://data.commoncrawl.org/) etc. and curated with [NeMo Curator](https://github.com/NVIDIA/NeMo-Curator/tree/main/tutorials/single_node_tutorial) + + +## Custom Tokenization for DAPT + +After placing the curated data in the directories mentioned above, we can proceed with custom tokenization and DAPT. + +* `./code/custom_tokenization.ipynb` walks through the custom tokenization workflow required for DAPT + +## Pretraining for DAPT + +Once we have the domain adapted custom tokenizer from above, we can proceed with pretraining using the customer tokenizer. + +* `./code/domain_adaptive_pretraining.ipynb` walks through the pretraining workflow required for DAPT diff --git a/tutorials/llm/llama/domain-adaptive-pretraining/code/__init__.py b/tutorials/llm/llama/domain-adaptive-pretraining/code/__init__.py new file mode 100644 index 000000000000..2e4094093d80 --- /dev/null +++ b/tutorials/llm/llama/domain-adaptive-pretraining/code/__init__.py @@ -0,0 +1,71 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo.collections.llm.recipes import ( + llama2_7b, + llama3_8b, + llama3_8b_16k, + llama3_8b_64k, + llama3_70b, + llama3_70b_16k, + llama3_70b_64k, + llama31_405b, + mistral, + mixtral_8x7b, + mixtral_8x7b_16k, + mixtral_8x7b_64k, + mixtral_8x22b, + nemotron, + nemotron3_4b, + nemotron3_8b, + nemotron4_15b, + nemotron4_15b_16k, + nemotron4_15b_64k, + nemotron4_22b, + nemotron4_22b_16k, + nemotron4_22b_64k, + nemotron4_340b, +) +from nemo.collections.llm.recipes.log.default import default_log, default_resume +from nemo.collections.llm.recipes.optim import adam + +__all__ = [ + "llama2_7b", + "llama3_8b", + "llama3_8b_16k", + "llama3_8b_64k", + "llama3_70b", + "llama3_70b_16k", + "llama3_70b_64k", + "llama31_405b", + "mistral", + "mixtral_8x7b", + "mixtral_8x7b_16k", + "mixtral_8x7b_64k", + "mixtral_8x22b", + "nemotron", + "nemotron3_4b", + "nemotron3_8b", + "nemotron4_15b", + "nemotron4_15b_16k", + "nemotron4_15b_64k", + "nemotron4_22b", + "nemotron4_22b_16k", + "nemotron4_22b_64k", + "nemotron4_340b", + "adam", + "default_log", + "default_resume", +] diff --git a/tutorials/llm/llama/domain-adaptive-pretraining/code/custom_tokenization.ipynb b/tutorials/llm/llama/domain-adaptive-pretraining/code/custom_tokenization.ipynb new file mode 100644 index 000000000000..f4f0547c59a2 --- /dev/null +++ b/tutorials/llm/llama/domain-adaptive-pretraining/code/custom_tokenization.ipynb @@ -0,0 +1,1920 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "6196c6f2-f088-4c28-b9a8-e921f9a7465d", + "metadata": {}, + "source": [ + "# Custom Tokenization for DAPT (Domain Adaptive Pre-Training)" + ] + }, + { + "cell_type": "markdown", + "id": "dbd33f30-2a18-480f-a7f1-210ac99b937c", + "metadata": {}, + "source": [ + "This notebook walks through the custom tokenization workflow required for DAPT (Domain Adaptive Pre-training) as shown in the schematic diagram below. \n", + "\n", + "![pipeline](imgs/tokenization_diagram.png)" + ] + }, + { + "cell_type": "markdown", + "id": "56f509da", + "metadata": {}, + "source": [ + "### Custom Tokenization Workflow" + ] + }, + { + "cell_type": "markdown", + "id": "12fe579a", + "metadata": {}, + "source": [ + "#### Goal\n", + "Given a pre-trained tokenizer trained on general purpose datasets (Original Tokenizer), our goal is to adapt it to a given domain that we want to apply it to (in this notebook, the example domain we are looking at is ChipDesign).\n", + "\n", + "When adapting a pre-trained tokenizer to a given domain, the main goals are to improve tokenization efficiency on domain-specific data, maintain efficiency and language model performance on general purpose datasets, and minimize the effort for retraining/fine-tuning. Since we don't have access to the entire general purpose data used for pretraining the original tokenizer, we want to preserve the existing token mappings, and any new tokens that are added should be strictly an \"extension\". \n", + "\n", + "Generally, when adapting tokenizer to domain-specific data, the goal is to create a tokenizer that is better suited to the vocabulary and structure of that specific domain. This can improve the efficiency and performance of the model on tasks within that domain through efficient representation of domain specific information.\n", + "\n", + "#### Approach \n", + "The general approach we adopt is to train a Domain Specific Tokenizer from scratch on domain data and use it to identify domain specific tokens that are missing from the original tokenizer. This is done by simply comparing the vocabs of the Original Tokenizer and the newly trained Domain Specific Tokenizer. The missing domain specific tokens are then added to the original tokenizer for extending it to get the final Domain Adapted Tokenizer. \n", + "\n", + "#### Tradeoff \n", + "However, there is a tradeoff to adding missing domain specific tokens to the Original Tokenizer. The challenge is to balance this tradeoff between tokenization efficiency on domain data vs disturbance to the performance on general-purpose data as a result of adding domain specific tokens to the Original Tokenizer.\n", + "\n", + "For instance, addition of a large no. of domain specific tokens can lead to higher efficiency on domain specific data, but DAPT process might take longer since it would take longer for the loss to converge​ due to disturbance of efficiency/performance on the general purpose data.\n", + "\n", + "On the other hand, addition of only a small no. of domain specific tokens can lead to maintained efficiency on general purpose data, but may lack coverage on the domain specific dataset​.\n", + "\n", + "#### Balancing The Tradeoff\n", + "To balance this tradeoff, instead of adding all identified missing domain specific tokens to the original tokenizer, we identify the most frequently occuring tokens using a threshold and only add the ones with usage frequencies above the given threshold to get the final Domain Adapted Tokenizer. \n", + "\n", + "For identifying the most frequently used tokens, we first extend the Original Tokenizer by adding all identified missing domain specific tokens to get an Extended Tokenizer. The Extended Tokenizer is then applyied to the domain specific data in order to identify high frequency tokens. Thus the Extended Tokenizer is just an intermediate step towards building a Domain Adapted Tokenizer.\n", + "\n", + "Finally, the Original Tokenizer is extended using only high frequency tokens to get the final Domain Adapted Tokenizer. " + ] + }, + { + "cell_type": "markdown", + "id": "d0b69b3b-aa66-42c9-b76f-2fde7f29a4b0", + "metadata": {}, + "source": [ + "## Notebook Outline\n", + "\n", + "To achieve the process described above, we’ve developed a step-by-step approach that this notebook will walk you through:\n", + "\n", + "- Step 0: Install pre-requisites and import the required modules\n", + "- Step 1: Download llama-2-70b embedding model and tokenizer (Original Tokenizer). Convert the orginal weights to trainable format and save. \n", + "- Step 2: Train an opt-350m tokenizer from scratch using domain-specific data to get a Domain Specific Tokenizer.\n", + "- Step 3: From the vocabulary of the newly trained tokenizer, identifying tokens that are absent in the general-purpose tokenizer and are rarely found in general-purpose datasets. Next, expand the general-purpose tokenizer with the newly identified tokens to get an Extended Tokenizer.\n", + "- Step 4: Apply the Extended Tokenizer to the domain-specific dataset, analyze the usage frequencies of the newly-added tokens, and select the top-K tokens in a way that their cumulative frequency accounts for approximately 98% (a hyper-parameter) of the total frequency of the new tokens.\n", + "- Step 5: Initialize the embeddings of the new tokens by utilizing the general-purpose tokenizer i.e., Original Tokenizer. When a new token is encountered, it is tokenized using the pretrained general-purpose tokenizer. The embedding and output layer weights corresponding to the new token are determined by averaging the embeddings / weights corresponding to the tokens generated by the general-purpose tokenizer.\n", + "- Step 6: Merge the new embeddings with the original embedding table (in llama2-2-70b) to get the final Domain Adapted Tokenizer.\n", + "## Data\n", + "\n", + "In this playbook, we will leverage chip domain/hardware datasets from open-source GitHub repositories, wiki URLs, and academic papers. Data has been processed and curated using [NeMo Curator](https://github.com/NVIDIA/NeMo-Curator/tree/main) as shown in this [playbook](https://github.com/jvamaraju/ndc_dapt_playbook/tree/dapt_jv)" + ] + }, + { + "cell_type": "markdown", + "id": "fbee82dd", + "metadata": {}, + "source": [ + "## NeMo Tools and Resources\n", + "\n", + "* [Nvidia Nemo Framework](https://github.com/NVIDIA/NeMo)" + ] + }, + { + "cell_type": "markdown", + "id": "74be8ece", + "metadata": {}, + "source": [ + "## Software Requirements\n", + "* Access to latest NeMo Framework NGC Containers\n", + "* This playbook has been tested on: nvcr.io/nvidia/nemo:24.07. It is expected to work similarly on other environments. " + ] + }, + { + "cell_type": "markdown", + "id": "d2b5ad09", + "metadata": {}, + "source": [ + "## Hardware Requirements\n", + "* This playbook can run on CPUs or GPUs. For GPUs, this playbook has been tested on minimum 1xA100 80G" + ] + }, + { + "cell_type": "markdown", + "id": "80bae538-308f-4d1b-8186-69de6226f3cd", + "metadata": {}, + "source": [ + "## Step 0: install the prerequisites and import the required modules" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cc83794a-daf9-44fb-9b89-f8cde05101a4", + "metadata": { + "scrolled": true, + "tags": [] + }, + "outputs": [], + "source": [ + "! pip install datasets sentencepiece jsonlines tokenizers transformers torch ftfy matplotlib\n", + "! pip install protobuf==3.20.1\n", + "! pip install --upgrade jupyter ipywidgets widgetsnbextension pandas-profiling" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "a91cd358", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "import torch\n", + "from datasets import Dataset\n", + "from datasets import IterableDataset\n", + "from datasets import load_dataset\n", + "import jsonlines\n", + "import glob\n", + "from tokenizers import (\n", + " decoders,\n", + " models,\n", + " normalizers,\n", + " pre_tokenizers,\n", + " processors,\n", + " trainers,\n", + " Tokenizer,\n", + ")\n", + "from transformers import AutoTokenizer\n", + "from tokenization_helper import *\n", + "from extend_tokenizer_utils import extend_tokenizer, extend_tokenizer_high_freq_tokens\n", + "from get_high_freq_tokens import get_high_freq_tokens\n", + "from util import load_weights, merge_embed" + ] + }, + { + "cell_type": "markdown", + "id": "24d00d02", + "metadata": {}, + "source": [ + "## Step 1: Download llama-2-70b embedding model and tokenizer (Original Tokenizer). Convert the orginal weights to trainable format and save. \n", + "\n", + "The Original Tokenizer model used here is the llama2 tokenizer which is a Byte Pair Encoding (BPE) model based on sentencepiece.\n", + "\n", + "Here we first log into the Hugging Face before downloading the model since the model is in a restricted repo." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "38cf0264", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Install the hugging face CLI\n", + "! pip install -U \"huggingface_hub[cli]\"\n", + "# Generate a user access token at https://huggingface.co/settings/tokens\n", + "\n", + "# To download the model, please login via huggingface-cli login since it is a restricted repo\n", + "! huggingface-cli login\n", + "# You will be prompted to enter your User Access Token. Copy and paste the token, then press Enter. The CLI will verify the token and save it locally." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "d54f21e5", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# create directory for storing the downloaded hugging face model \n", + "os.makedirs(\"models/weight/llama2-hf\", exist_ok=True)\n", + "\n", + "# create directories for storing the model weights \n", + "os.makedirs(\"models/weight/llama2/ori_llama2-hf_weight\", exist_ok=True)\n", + "os.makedirs(\"models/weight/llama2/new_llama2-hf_weight\", exist_ok=True)\n", + "\n", + "# create directories for storing the tokenizers\n", + "os.makedirs(\"models/tokenizer/llama2/original_tokenizer\", exist_ok=True)\n", + "os.makedirs(\"models/tokenizer/llama2/new_tokenizer\", exist_ok=True)" + ] + }, + { + "cell_type": "markdown", + "id": "558753ac", + "metadata": {}, + "source": [ + "Before running the next step, make sure you have access granted for Meta's Llama2 models gated group. You can fill the form available on https://huggingface.co/meta-llama/Llama-2-7b in order to get the access. (Takes ~20 minutes)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "7f0c3988", + "metadata": { + "scrolled": true, + "tags": [] + }, + "outputs": [], + "source": [ + "# download llama2-70b model weights and tokenizer \n", + "! huggingface-cli download meta-llama/Llama-2-70b --local-dir ./models/weight/llama2-hf/\n", + "\n", + "# #Copy original tokenizer to a different folder\n", + "! cp ./models/weight/llama2-hf/tokenizer.model ./models/tokenizer/llama2/original_tokenizer\n", + "\n", + "# Load embedding and output layer weights (size = (vocab size,embedding dim)) from each snapshot and create a dict\n", + "load_path = \"./models/weight/llama2-hf\"\n", + "save_path = './models/weight/llama2/ori_llama2-hf_weight'\n", + "\n", + "if not os.path.exists(save_path):\n", + " os.makedirs(save_path)\n", + " \n", + "#load weight and store in a dictionary suitable for NeMo\n", + "load_weights(load_path, save_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "6ade9a02-d38f-436c-82d0-bd16b54dbbf8", + "metadata": { + "scrolled": true, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Index: 0, layer: tok_embeddings.weight, Layer size: torch.Size([32000, 1024])\n", + "Index: 1, layer: norm.weight, Layer size: torch.Size([8192])\n", + "Index: 2, layer: output.weight, Layer size: torch.Size([4000, 8192])\n", + "Index: 3, layer: layers.0.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 4, layer: layers.0.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 5, layer: layers.0.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 6, layer: layers.0.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 7, layer: layers.0.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 8, layer: layers.0.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 9, layer: layers.0.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 10, layer: layers.0.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 11, layer: layers.0.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 12, layer: layers.1.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 13, layer: layers.1.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 14, layer: layers.1.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 15, layer: layers.1.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 16, layer: layers.1.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 17, layer: layers.1.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 18, layer: layers.1.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 19, layer: layers.1.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 20, layer: layers.1.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 21, layer: layers.2.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 22, layer: layers.2.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 23, layer: layers.2.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 24, layer: layers.2.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 25, layer: layers.2.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 26, layer: layers.2.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 27, layer: layers.2.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 28, layer: layers.2.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 29, layer: layers.2.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 30, layer: layers.3.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 31, layer: layers.3.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 32, layer: layers.3.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 33, layer: layers.3.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 34, layer: layers.3.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 35, layer: layers.3.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 36, layer: layers.3.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 37, layer: layers.3.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 38, layer: layers.3.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 39, layer: layers.4.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 40, layer: layers.4.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 41, layer: layers.4.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 42, layer: layers.4.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 43, layer: layers.4.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 44, layer: layers.4.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 45, layer: layers.4.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 46, layer: layers.4.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 47, layer: layers.4.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 48, layer: layers.5.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 49, layer: layers.5.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 50, layer: layers.5.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 51, layer: layers.5.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 52, layer: layers.5.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 53, layer: layers.5.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 54, layer: layers.5.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 55, layer: layers.5.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 56, layer: layers.5.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 57, layer: layers.6.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 58, layer: layers.6.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 59, layer: layers.6.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 60, layer: layers.6.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 61, layer: layers.6.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 62, layer: layers.6.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 63, layer: layers.6.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 64, layer: layers.6.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 65, layer: layers.6.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 66, layer: layers.7.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 67, layer: layers.7.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 68, layer: layers.7.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 69, layer: layers.7.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 70, layer: layers.7.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 71, layer: layers.7.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 72, layer: layers.7.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 73, layer: layers.7.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 74, layer: layers.7.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 75, layer: layers.8.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 76, layer: layers.8.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 77, layer: layers.8.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 78, layer: layers.8.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 79, layer: layers.8.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 80, layer: layers.8.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 81, layer: layers.8.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 82, layer: layers.8.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 83, layer: layers.8.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 84, layer: layers.9.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 85, layer: layers.9.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 86, layer: layers.9.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 87, layer: layers.9.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 88, layer: layers.9.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 89, layer: layers.9.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 90, layer: layers.9.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 91, layer: layers.9.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 92, layer: layers.9.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 93, layer: layers.10.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 94, layer: layers.10.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 95, layer: layers.10.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 96, layer: layers.10.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 97, layer: layers.10.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 98, layer: layers.10.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 99, layer: layers.10.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 100, layer: layers.10.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 101, layer: layers.10.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 102, layer: layers.11.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 103, layer: layers.11.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 104, layer: layers.11.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 105, layer: layers.11.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 106, layer: layers.11.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 107, layer: layers.11.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 108, layer: layers.11.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 109, layer: layers.11.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 110, layer: layers.11.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 111, layer: layers.12.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 112, layer: layers.12.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 113, layer: layers.12.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 114, layer: layers.12.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 115, layer: layers.12.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 116, layer: layers.12.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 117, layer: layers.12.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 118, layer: layers.12.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 119, layer: layers.12.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 120, layer: layers.13.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 121, layer: layers.13.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 122, layer: layers.13.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 123, layer: layers.13.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 124, layer: layers.13.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 125, layer: layers.13.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 126, layer: layers.13.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 127, layer: layers.13.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 128, layer: layers.13.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 129, layer: layers.14.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 130, layer: layers.14.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 131, layer: layers.14.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 132, layer: layers.14.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 133, layer: layers.14.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 134, layer: layers.14.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 135, layer: layers.14.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 136, layer: layers.14.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 137, layer: layers.14.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 138, layer: layers.15.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 139, layer: layers.15.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 140, layer: layers.15.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 141, layer: layers.15.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 142, layer: layers.15.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 143, layer: layers.15.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 144, layer: layers.15.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 145, layer: layers.15.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 146, layer: layers.15.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 147, layer: layers.16.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 148, layer: layers.16.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 149, layer: layers.16.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 150, layer: layers.16.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 151, layer: layers.16.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 152, layer: layers.16.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 153, layer: layers.16.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 154, layer: layers.16.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 155, layer: layers.16.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 156, layer: layers.17.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 157, layer: layers.17.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 158, layer: layers.17.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 159, layer: layers.17.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 160, layer: layers.17.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 161, layer: layers.17.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 162, layer: layers.17.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 163, layer: layers.17.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 164, layer: layers.17.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 165, layer: layers.18.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 166, layer: layers.18.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 167, layer: layers.18.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 168, layer: layers.18.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 169, layer: layers.18.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 170, layer: layers.18.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 171, layer: layers.18.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 172, layer: layers.18.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 173, layer: layers.18.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 174, layer: layers.19.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 175, layer: layers.19.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 176, layer: layers.19.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 177, layer: layers.19.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 178, layer: layers.19.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 179, layer: layers.19.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 180, layer: layers.19.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 181, layer: layers.19.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 182, layer: layers.19.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 183, layer: layers.20.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 184, layer: layers.20.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 185, layer: layers.20.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 186, layer: layers.20.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 187, layer: layers.20.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 188, layer: layers.20.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 189, layer: layers.20.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 190, layer: layers.20.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 191, layer: layers.20.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 192, layer: layers.21.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 193, layer: layers.21.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 194, layer: layers.21.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 195, layer: layers.21.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 196, layer: layers.21.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 197, layer: layers.21.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 198, layer: layers.21.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 199, layer: layers.21.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 200, layer: layers.21.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 201, layer: layers.22.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 202, layer: layers.22.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 203, layer: layers.22.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 204, layer: layers.22.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 205, layer: layers.22.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 206, layer: layers.22.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 207, layer: layers.22.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 208, layer: layers.22.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 209, layer: layers.22.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 210, layer: layers.23.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 211, layer: layers.23.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 212, layer: layers.23.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 213, layer: layers.23.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 214, layer: layers.23.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 215, layer: layers.23.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 216, layer: layers.23.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 217, layer: layers.23.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 218, layer: layers.23.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 219, layer: layers.24.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 220, layer: layers.24.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 221, layer: layers.24.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 222, layer: layers.24.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 223, layer: layers.24.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 224, layer: layers.24.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 225, layer: layers.24.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 226, layer: layers.24.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 227, layer: layers.24.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 228, layer: layers.25.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 229, layer: layers.25.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 230, layer: layers.25.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 231, layer: layers.25.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 232, layer: layers.25.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 233, layer: layers.25.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 234, layer: layers.25.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 235, layer: layers.25.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 236, layer: layers.25.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 237, layer: layers.26.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 238, layer: layers.26.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 239, layer: layers.26.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 240, layer: layers.26.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 241, layer: layers.26.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 242, layer: layers.26.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 243, layer: layers.26.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 244, layer: layers.26.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 245, layer: layers.26.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 246, layer: layers.27.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 247, layer: layers.27.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 248, layer: layers.27.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 249, layer: layers.27.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 250, layer: layers.27.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 251, layer: layers.27.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 252, layer: layers.27.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 253, layer: layers.27.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 254, layer: layers.27.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 255, layer: layers.28.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 256, layer: layers.28.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 257, layer: layers.28.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 258, layer: layers.28.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 259, layer: layers.28.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 260, layer: layers.28.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 261, layer: layers.28.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 262, layer: layers.28.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 263, layer: layers.28.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 264, layer: layers.29.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 265, layer: layers.29.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 266, layer: layers.29.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 267, layer: layers.29.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 268, layer: layers.29.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 269, layer: layers.29.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 270, layer: layers.29.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 271, layer: layers.29.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 272, layer: layers.29.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 273, layer: layers.30.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 274, layer: layers.30.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 275, layer: layers.30.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 276, layer: layers.30.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 277, layer: layers.30.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 278, layer: layers.30.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 279, layer: layers.30.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 280, layer: layers.30.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 281, layer: layers.30.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 282, layer: layers.31.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 283, layer: layers.31.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 284, layer: layers.31.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 285, layer: layers.31.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 286, layer: layers.31.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 287, layer: layers.31.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 288, layer: layers.31.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 289, layer: layers.31.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 290, layer: layers.31.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 291, layer: layers.32.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 292, layer: layers.32.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 293, layer: layers.32.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 294, layer: layers.32.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 295, layer: layers.32.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 296, layer: layers.32.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 297, layer: layers.32.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 298, layer: layers.32.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 299, layer: layers.32.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 300, layer: layers.33.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 301, layer: layers.33.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 302, layer: layers.33.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 303, layer: layers.33.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 304, layer: layers.33.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 305, layer: layers.33.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 306, layer: layers.33.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 307, layer: layers.33.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 308, layer: layers.33.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 309, layer: layers.34.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 310, layer: layers.34.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 311, layer: layers.34.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 312, layer: layers.34.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 313, layer: layers.34.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 314, layer: layers.34.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 315, layer: layers.34.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 316, layer: layers.34.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 317, layer: layers.34.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 318, layer: layers.35.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 319, layer: layers.35.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 320, layer: layers.35.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 321, layer: layers.35.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 322, layer: layers.35.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 323, layer: layers.35.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 324, layer: layers.35.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 325, layer: layers.35.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 326, layer: layers.35.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 327, layer: layers.36.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 328, layer: layers.36.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 329, layer: layers.36.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 330, layer: layers.36.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 331, layer: layers.36.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 332, layer: layers.36.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 333, layer: layers.36.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 334, layer: layers.36.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 335, layer: layers.36.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 336, layer: layers.37.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 337, layer: layers.37.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 338, layer: layers.37.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 339, layer: layers.37.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 340, layer: layers.37.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 341, layer: layers.37.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 342, layer: layers.37.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 343, layer: layers.37.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 344, layer: layers.37.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 345, layer: layers.38.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 346, layer: layers.38.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 347, layer: layers.38.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 348, layer: layers.38.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 349, layer: layers.38.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 350, layer: layers.38.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 351, layer: layers.38.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 352, layer: layers.38.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 353, layer: layers.38.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 354, layer: layers.39.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 355, layer: layers.39.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 356, layer: layers.39.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 357, layer: layers.39.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 358, layer: layers.39.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 359, layer: layers.39.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 360, layer: layers.39.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 361, layer: layers.39.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 362, layer: layers.39.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 363, layer: layers.40.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 364, layer: layers.40.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 365, layer: layers.40.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 366, layer: layers.40.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 367, layer: layers.40.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 368, layer: layers.40.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 369, layer: layers.40.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 370, layer: layers.40.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 371, layer: layers.40.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 372, layer: layers.41.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 373, layer: layers.41.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 374, layer: layers.41.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 375, layer: layers.41.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 376, layer: layers.41.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 377, layer: layers.41.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 378, layer: layers.41.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 379, layer: layers.41.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 380, layer: layers.41.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 381, layer: layers.42.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 382, layer: layers.42.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 383, layer: layers.42.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 384, layer: layers.42.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 385, layer: layers.42.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 386, layer: layers.42.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 387, layer: layers.42.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 388, layer: layers.42.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 389, layer: layers.42.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 390, layer: layers.43.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 391, layer: layers.43.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 392, layer: layers.43.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 393, layer: layers.43.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 394, layer: layers.43.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 395, layer: layers.43.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 396, layer: layers.43.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 397, layer: layers.43.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 398, layer: layers.43.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 399, layer: layers.44.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 400, layer: layers.44.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 401, layer: layers.44.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 402, layer: layers.44.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 403, layer: layers.44.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 404, layer: layers.44.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 405, layer: layers.44.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 406, layer: layers.44.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 407, layer: layers.44.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 408, layer: layers.45.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 409, layer: layers.45.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 410, layer: layers.45.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 411, layer: layers.45.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 412, layer: layers.45.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 413, layer: layers.45.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 414, layer: layers.45.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 415, layer: layers.45.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 416, layer: layers.45.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 417, layer: layers.46.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 418, layer: layers.46.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 419, layer: layers.46.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 420, layer: layers.46.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 421, layer: layers.46.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 422, layer: layers.46.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 423, layer: layers.46.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 424, layer: layers.46.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 425, layer: layers.46.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 426, layer: layers.47.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 427, layer: layers.47.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 428, layer: layers.47.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 429, layer: layers.47.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 430, layer: layers.47.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 431, layer: layers.47.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 432, layer: layers.47.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 433, layer: layers.47.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 434, layer: layers.47.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 435, layer: layers.48.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 436, layer: layers.48.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 437, layer: layers.48.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 438, layer: layers.48.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 439, layer: layers.48.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 440, layer: layers.48.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 441, layer: layers.48.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 442, layer: layers.48.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 443, layer: layers.48.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 444, layer: layers.49.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 445, layer: layers.49.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 446, layer: layers.49.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 447, layer: layers.49.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 448, layer: layers.49.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 449, layer: layers.49.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 450, layer: layers.49.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 451, layer: layers.49.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 452, layer: layers.49.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 453, layer: layers.50.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 454, layer: layers.50.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 455, layer: layers.50.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 456, layer: layers.50.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 457, layer: layers.50.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 458, layer: layers.50.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 459, layer: layers.50.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 460, layer: layers.50.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 461, layer: layers.50.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 462, layer: layers.51.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 463, layer: layers.51.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 464, layer: layers.51.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 465, layer: layers.51.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 466, layer: layers.51.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 467, layer: layers.51.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 468, layer: layers.51.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 469, layer: layers.51.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 470, layer: layers.51.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 471, layer: layers.52.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 472, layer: layers.52.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 473, layer: layers.52.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 474, layer: layers.52.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 475, layer: layers.52.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 476, layer: layers.52.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 477, layer: layers.52.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 478, layer: layers.52.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 479, layer: layers.52.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 480, layer: layers.53.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 481, layer: layers.53.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 482, layer: layers.53.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 483, layer: layers.53.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 484, layer: layers.53.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 485, layer: layers.53.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 486, layer: layers.53.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 487, layer: layers.53.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 488, layer: layers.53.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 489, layer: layers.54.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 490, layer: layers.54.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 491, layer: layers.54.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 492, layer: layers.54.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 493, layer: layers.54.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 494, layer: layers.54.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 495, layer: layers.54.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 496, layer: layers.54.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 497, layer: layers.54.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 498, layer: layers.55.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 499, layer: layers.55.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 500, layer: layers.55.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 501, layer: layers.55.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 502, layer: layers.55.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 503, layer: layers.55.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 504, layer: layers.55.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 505, layer: layers.55.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 506, layer: layers.55.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 507, layer: layers.56.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 508, layer: layers.56.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 509, layer: layers.56.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 510, layer: layers.56.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 511, layer: layers.56.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 512, layer: layers.56.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 513, layer: layers.56.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 514, layer: layers.56.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 515, layer: layers.56.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 516, layer: layers.57.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 517, layer: layers.57.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 518, layer: layers.57.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 519, layer: layers.57.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 520, layer: layers.57.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 521, layer: layers.57.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 522, layer: layers.57.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 523, layer: layers.57.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 524, layer: layers.57.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 525, layer: layers.58.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 526, layer: layers.58.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 527, layer: layers.58.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 528, layer: layers.58.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 529, layer: layers.58.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 530, layer: layers.58.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 531, layer: layers.58.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 532, layer: layers.58.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 533, layer: layers.58.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 534, layer: layers.59.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 535, layer: layers.59.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 536, layer: layers.59.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 537, layer: layers.59.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 538, layer: layers.59.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 539, layer: layers.59.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 540, layer: layers.59.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 541, layer: layers.59.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 542, layer: layers.59.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 543, layer: layers.60.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 544, layer: layers.60.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 545, layer: layers.60.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 546, layer: layers.60.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 547, layer: layers.60.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 548, layer: layers.60.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 549, layer: layers.60.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 550, layer: layers.60.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 551, layer: layers.60.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 552, layer: layers.61.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 553, layer: layers.61.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 554, layer: layers.61.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 555, layer: layers.61.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 556, layer: layers.61.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 557, layer: layers.61.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 558, layer: layers.61.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 559, layer: layers.61.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 560, layer: layers.61.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 561, layer: layers.62.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 562, layer: layers.62.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 563, layer: layers.62.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 564, layer: layers.62.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 565, layer: layers.62.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 566, layer: layers.62.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 567, layer: layers.62.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 568, layer: layers.62.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 569, layer: layers.62.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 570, layer: layers.63.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 571, layer: layers.63.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 572, layer: layers.63.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 573, layer: layers.63.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 574, layer: layers.63.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 575, layer: layers.63.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 576, layer: layers.63.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 577, layer: layers.63.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 578, layer: layers.63.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 579, layer: layers.64.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 580, layer: layers.64.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 581, layer: layers.64.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 582, layer: layers.64.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 583, layer: layers.64.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 584, layer: layers.64.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 585, layer: layers.64.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 586, layer: layers.64.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 587, layer: layers.64.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 588, layer: layers.65.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 589, layer: layers.65.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 590, layer: layers.65.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 591, layer: layers.65.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 592, layer: layers.65.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 593, layer: layers.65.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 594, layer: layers.65.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 595, layer: layers.65.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 596, layer: layers.65.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 597, layer: layers.66.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 598, layer: layers.66.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 599, layer: layers.66.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 600, layer: layers.66.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 601, layer: layers.66.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 602, layer: layers.66.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 603, layer: layers.66.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 604, layer: layers.66.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 605, layer: layers.66.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 606, layer: layers.67.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 607, layer: layers.67.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 608, layer: layers.67.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 609, layer: layers.67.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 610, layer: layers.67.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 611, layer: layers.67.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 612, layer: layers.67.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 613, layer: layers.67.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 614, layer: layers.67.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 615, layer: layers.68.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 616, layer: layers.68.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 617, layer: layers.68.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 618, layer: layers.68.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 619, layer: layers.68.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 620, layer: layers.68.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 621, layer: layers.68.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 622, layer: layers.68.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 623, layer: layers.68.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 624, layer: layers.69.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 625, layer: layers.69.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 626, layer: layers.69.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 627, layer: layers.69.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 628, layer: layers.69.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 629, layer: layers.69.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 630, layer: layers.69.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 631, layer: layers.69.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 632, layer: layers.69.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 633, layer: layers.70.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 634, layer: layers.70.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 635, layer: layers.70.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 636, layer: layers.70.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 637, layer: layers.70.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 638, layer: layers.70.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 639, layer: layers.70.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 640, layer: layers.70.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 641, layer: layers.70.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 642, layer: layers.71.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 643, layer: layers.71.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 644, layer: layers.71.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 645, layer: layers.71.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 646, layer: layers.71.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 647, layer: layers.71.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 648, layer: layers.71.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 649, layer: layers.71.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 650, layer: layers.71.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 651, layer: layers.72.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 652, layer: layers.72.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 653, layer: layers.72.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 654, layer: layers.72.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 655, layer: layers.72.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 656, layer: layers.72.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 657, layer: layers.72.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 658, layer: layers.72.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 659, layer: layers.72.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 660, layer: layers.73.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 661, layer: layers.73.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 662, layer: layers.73.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 663, layer: layers.73.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 664, layer: layers.73.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 665, layer: layers.73.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 666, layer: layers.73.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 667, layer: layers.73.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 668, layer: layers.73.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 669, layer: layers.74.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 670, layer: layers.74.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 671, layer: layers.74.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 672, layer: layers.74.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 673, layer: layers.74.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 674, layer: layers.74.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 675, layer: layers.74.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 676, layer: layers.74.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 677, layer: layers.74.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 678, layer: layers.75.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 679, layer: layers.75.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 680, layer: layers.75.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 681, layer: layers.75.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 682, layer: layers.75.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 683, layer: layers.75.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 684, layer: layers.75.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 685, layer: layers.75.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 686, layer: layers.75.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 687, layer: layers.76.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 688, layer: layers.76.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 689, layer: layers.76.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 690, layer: layers.76.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 691, layer: layers.76.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 692, layer: layers.76.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 693, layer: layers.76.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 694, layer: layers.76.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 695, layer: layers.76.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 696, layer: layers.77.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 697, layer: layers.77.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 698, layer: layers.77.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 699, layer: layers.77.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 700, layer: layers.77.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 701, layer: layers.77.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 702, layer: layers.77.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 703, layer: layers.77.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 704, layer: layers.77.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 705, layer: layers.78.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 706, layer: layers.78.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 707, layer: layers.78.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 708, layer: layers.78.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 709, layer: layers.78.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 710, layer: layers.78.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 711, layer: layers.78.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 712, layer: layers.78.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 713, layer: layers.78.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 714, layer: layers.79.attention.wq.weight, Layer size: torch.Size([1024, 8192])\n", + "Index: 715, layer: layers.79.attention.wk.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 716, layer: layers.79.attention.wv.weight, Layer size: torch.Size([128, 8192])\n", + "Index: 717, layer: layers.79.attention.wo.weight, Layer size: torch.Size([8192, 1024])\n", + "Index: 718, layer: layers.79.feed_forward.w1.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 719, layer: layers.79.feed_forward.w2.weight, Layer size: torch.Size([8192, 3584])\n", + "Index: 720, layer: layers.79.feed_forward.w3.weight, Layer size: torch.Size([3584, 8192])\n", + "Index: 721, layer: layers.79.attention_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 722, layer: layers.79.ffn_norm.weight, Layer size: torch.Size([8192])\n", + "Index: 723, layer: rope.freqs, Layer size: torch.Size([64])\n" + ] + } + ], + "source": [ + "# check layers and dimensions (optional)\n", + "state_dict = torch.load(f\"{load_path}/consolidated.0{1}.pth\")\n", + "for index, (key, value) in enumerate(state_dict.items()):\n", + " print(f\"Index: {index}, layer: {key}, Layer size: {value.size()}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a5a65e1f", + "metadata": {}, + "source": [ + "## Step 2: Train a tokenizer from scratch using domain-specific data to get a Domain Specific Tokenizer." + ] + }, + { + "cell_type": "markdown", + "id": "fa45b02d-2e2f-4b81-9cba-ed07cbda5b9d", + "metadata": {}, + "source": [ + "First, we train a tokenizer from scratch using domain-specific data.\n", + "\n", + "The tokenizer that we use is the facebook/opt-350m model tokenizer available here on hugging face. Similar to the llama-2 tokenizer, opt-350m tokenizer is also a Byte Pair Encoding (BPE) model and since we are training from scratch we could use any of them. Infact, we can use any model's tokenizer that is implemented based on BPE since the training algorithm inside the tokenizer is what matters. However, we chose opt-350m since it has a more general purpose design and can be used flexibly across different tasks/domains and with various models beyond the OPT series. On the other hand llama-2 tokenizer is designed specifically for llama-2 architecture, optimizing performance for tasks that llama-2 model is intended to handle. \n", + "\n", + "The two hyperparameters that need to be set here are ```batch_size``` and ```vocab_size```.
\n", + "\n", + "```vocab_size``` : is the target vocab size in finetuning the tokenizer. This depends on the original tokenizer and should be slightly higher than half of the original vocab size. Note that this doesn't have to equal the number of new tokens that will be added. \n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "518ca72b-4ab8-4538-8eb4-42d648346347", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Is a directory: True\n" + ] + } + ], + "source": [ + "data_root = \"./data/all_jsonl_data_sample/\" # path where the domain specific data is stored\n", + "save_root = \"./models/tokenizer/llama2/\" # path to save the finetuned opt tokenizer\n", + "batch_size = 1000 # batch size used in the tokenization process\n", + "vocab_size = 20000 # target vocab size for training opt tokenizer\n", + "\n", + "# ensure that the directory exists before changing permissions\n", + "directory = \"../code/\"\n", + "is_directory = os.path.isdir(directory)\n", + "print(f\"Is a directory: {is_directory}\")\n", + "\n", + "# change permissions to ensure we have read, write and execute permissions\n", + "! chmod ugo+rwx ../code/" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "519e6c13-45c3-4f52-9ed0-770d4ec62766", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:797: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Before Training: \n", + "total token cnt 66025\n", + "\n", + "\n", + "\n", + "After Training: \n", + "total token cnt 47712\n" + ] + }, + { + "data": { + "text/plain": [ + "('./models/tokenizer/llama2/custom_tokenizer_init_20000_json/tokenizer_config.json',\n", + " './models/tokenizer/llama2/custom_tokenizer_init_20000_json/special_tokens_map.json',\n", + " './models/tokenizer/llama2/custom_tokenizer_init_20000_json/vocab.json',\n", + " './models/tokenizer/llama2/custom_tokenizer_init_20000_json/merges.txt',\n", + " './models/tokenizer/llama2/custom_tokenizer_init_20000_json/added_tokens.json',\n", + " './models/tokenizer/llama2/custom_tokenizer_init_20000_json/tokenizer.json')" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Train a tokenizer from scratch and save output files\n", + "keys = [\"text\"] # keys to extract from json files\n", + "tokenizer = AutoTokenizer.from_pretrained(\"facebook/opt-350m\") # load pre-trained tokenizer (https://huggingface.co/facebook/opt-350m)\n", + "# Train the tokenizer from scratch on a new corpus with the same defaults (in terms of special tokens or tokenization pipeline) as the current one.\n", + "tokenizer = train_tokenizer(data_root, batch_size, vocab_size, tokenizer, keys)\n", + "\n", + "#Save and print paths\n", + "tokenizer.save_pretrained(save_root + \"custom_tokenizer_init_\" + str(vocab_size) + \"_json\")" + ] + }, + { + "cell_type": "markdown", + "id": "127e0591-fbaa-41bc-87a5-f594587ea12d", + "metadata": { + "tags": [] + }, + "source": [ + "## Step 3: From the vocabulary of the newly trained tokenizer, identify tokens that are absent in the general-purpose tokenizer and are rarely found in general-purpose datasets. Next, expand the general-purpose tokenizer with the newly identified tokens to get an extended Tokenizer.\n", + "\n", + "Here we expand/resize the model embeddings of the original general-purpose tokenizer with the newly identified tokens in Step 3 to get an extended tokenizer.\n", + "\n", + "The two hyperparemeters that need to be set here are ```split``` and ```model_type```. \n", + "\n", + "```split```: is the number of partitions to split the embeddings in (.pt files) for the purpose of model parallelism.\n", + "\n", + "```model_type``` : this is the original tokenizer model (llama2 in our case)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "ec202bf4-3508-4a64-90c6-debcf116e81e", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Domain vocab size: 5965\n", + "token pattern: [a-zA-Z]\n", + "Num of added tokens and dropped tokens 4931 1034\n", + "Original model pieces: 32000\n", + "input: \"/large_experiments/theorem/datasets/MERGED/all.test1.merged\"\n", + "model_prefix: \"spm_model_32k_200M_charcov099995_allowWSO__v2\"\n", + "model_type: BPE\n", + "vocab_size: 32000\n", + "self_test_sample_size: 0\n", + "input_format: \"text\"\n", + "character_coverage: 0.99995\n", + "input_sentence_size: 200000000\n", + "seed_sentencepiece_size: 1000000\n", + "shrinking_factor: 0.75\n", + "num_threads: 80\n", + "num_sub_iterations: 2\n", + "max_sentence_length: 4192\n", + "shuffle_input_sentence: true\n", + "max_sentencepiece_length: 16\n", + "split_by_unicode_script: true\n", + "split_by_whitespace: true\n", + "split_by_number: true\n", + "treat_whitespace_as_suffix: false\n", + "split_digits: true\n", + "allow_whitespace_only_pieces: true\n", + "vocabulary_output_piece_score: true\n", + "hard_vocab_limit: true\n", + "use_all_vocab: false\n", + "byte_fallback: true\n", + "required_chars: \"\"\n", + "unk_id: 0\n", + "bos_id: 1\n", + "eos_id: 2\n", + "pad_id: -1\n", + "unk_surface: \" \\342\\201\\207 \"\n", + "unk_piece: \"\"\n", + "bos_piece: \"\"\n", + "eos_piece: \"\"\n", + "pad_piece: \"\"\n", + "train_extremely_large_corpus: false\n", + "enable_differential_privacy: false\n", + "differential_privacy_noise_level: 0.0\n", + "differential_privacy_clipping_threshold: 0\n", + "\n", + "original vocab size: 32000\n", + "new token cnt: 1400\n", + "add token cnt: 2048\n", + "add normal token cnt: 1400\n", + "add dummy token cnt: 648\n", + "New model pieces: 34048\n", + "input: \"/large_experiments/theorem/datasets/MERGED/all.test1.merged\"\n", + "model_prefix: \"spm_model_32k_200M_charcov099995_allowWSO__v2\"\n", + "model_type: BPE\n", + "vocab_size: 32000\n", + "self_test_sample_size: 0\n", + "input_format: \"text\"\n", + "character_coverage: 0.99995\n", + "input_sentence_size: 200000000\n", + "seed_sentencepiece_size: 1000000\n", + "shrinking_factor: 0.75\n", + "num_threads: 80\n", + "num_sub_iterations: 2\n", + "max_sentence_length: 4192\n", + "shuffle_input_sentence: true\n", + "max_sentencepiece_length: 16\n", + "split_by_unicode_script: true\n", + "split_by_whitespace: true\n", + "split_by_number: true\n", + "treat_whitespace_as_suffix: false\n", + "split_digits: true\n", + "allow_whitespace_only_pieces: true\n", + "vocabulary_output_piece_score: true\n", + "hard_vocab_limit: true\n", + "use_all_vocab: false\n", + "byte_fallback: true\n", + "required_chars: \"\"\n", + "unk_id: 0\n", + "bos_id: 1\n", + "eos_id: 2\n", + "pad_id: -1\n", + "unk_surface: \" \\342\\201\\207 \"\n", + "unk_piece: \"\"\n", + "bos_piece: \"\"\n", + "eos_piece: \"\"\n", + "pad_piece: \"\"\n", + "train_extremely_large_corpus: false\n", + "enable_differential_privacy: false\n", + "differential_privacy_noise_level: 0.0\n", + "differential_privacy_clipping_threshold: 0\n", + "\n", + "Parent directory './models/tokenizer/llama2/new_tokenizer' exists.\n", + "Parent directory './models/tokenizer/llama2/new_tokenizer' exists.\n", + "word_embedding shape: torch.Size([32000, 8192])\n", + "output_layer shape: torch.Size([32000, 8192])\n", + "Parent directory './models/weight/llama2/new_llama2-hf_weight' exists.\n", + "Parent directory './models/weight/llama2/new_llama2-hf_weight' exists.\n", + "Parent directory './models/weight/llama2/new_llama2-hf_weight' exists.\n", + "Parent directory './models/weight/llama2/new_llama2-hf_weight' exists.\n", + "Parent directory './models/weight/llama2/new_llama2-hf_weight' exists.\n", + "Parent directory './models/weight/llama2/new_llama2-hf_weight' exists.\n", + "Parent directory './models/weight/llama2/new_llama2-hf_weight' exists.\n", + "Parent directory './models/weight/llama2/new_llama2-hf_weight' exists.\n", + "Completed saving new embeddings\n", + "Vocabulary path for extended tokenizer: ./models/tokenizer/llama2/new_tokenizer/code_gen_vocab.json\n", + "Tokenizer model path for extended tokenizer: ./models/tokenizer/llama2/new_tokenizer/tokenizer_code_gen.model\n", + "Modified embedding weights path for extended tokenizer: ./models/weight/llama2/new_llama2-hf_weight/\n" + ] + } + ], + "source": [ + "split = 8 # number of partitions to split the embeddings of domain-adapted tokenizer\n", + "model_type = \"llama2\" # Add more model_types if you want the codebase to support alternate ones\n", + "extend_tokenizer(vocab_size, split, model_type)" + ] + }, + { + "cell_type": "markdown", + "id": "82ddf5a4-663f-40b0-9477-bd3d3e803c12", + "metadata": {}, + "source": [ + "## Step 4: Use the extended Tokenizer to anylze the frequency of newly added tokens" + ] + }, + { + "cell_type": "markdown", + "id": "9c78a7b5-8122-4d95-afd1-997f047c37f1", + "metadata": {}, + "source": [ + "Here we apply the extended tokenizer to the domain-specific dataset, analyzing the usage frequencies of the newly-added tokens, and selecting the top-K tokens in a way that their cumulative frequency accounts for approximately 98% (a hyper-parameter: ```freq_threshold```) of the total frequency of the new tokens.\n", + "\n", + "The idea is that only high-frequency tokens will be added to the vocabulary of the original tokenizer to get the final domain adapted tokenizer. \n", + "\n", + "The benefits of high-frequency token analysis have been explored in several studies: ([Liu, Mingjie, et al](https://research.nvidia.com/publication/2023-10_chipnemo-domain-adapted-llms-chip-design); [Lian, Haoran, et al](https://arxiv.org/abs/2404.17808)).This is because previous studies have shown that disparities in token frequencies can result in imbalanced learning difficulties across different tokens. For instance, low frequency tokens are harder to learn for models ([Su, Zhenpeng, et al](https://arxiv.org/abs/2310.19531); [Lin, Tsung-Yi, et al](https://openaccess.thecvf.com/content_iccv_2017/html/Lin_Focal_Loss_for_ICCV_2017_paper.html)).\n", + "\n", + "We use two functions for frequency analysis. Helper function `analyze_token_usage` applies the extended tokenizer to domain specific data, and stores the usage/occurence frequencies of the newly-added tokens at `token_usage_path`.
\n", + "\n", + "Helper function `get_high_freq_tokens` looks at the token usage frequencies from above and performs a binary search to search for domain specific tokens with usage frequency above the specified threshold (`freq_threshold` parameter). It stores the tokens it finds at `high_freq_tokens_path`." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "b36a5465-b409-4d32-9a2f-1dd7c91ef917", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "split = 8 # number of partitions to split the embeddings of domain-adapted tokenizer\n", + "model_type = \"llama2\"\n", + "tag = \"code_gen\"\n", + "keys = [\"text\"]\n", + "# path to the saved extended tokenizer (from previous tep)\n", + "extended_tokenizer_path = f\"./models/tokenizer/{model_type}/new_tokenizer/tokenizer_{tag}.model\"\n", + "# path to save token usage frequency analysis results\n", + "token_usage_path = f\"./models/tokenizer/{model_type}/new_tokenizer/{model_type}_token_usage.json\"" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "7e2b40db-82d6-48ca-a900-145647b4dff1", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "vocab_size: 34048\n", + "ori cnt and new cnt: 2209.0 22.0\n", + "ori cnt and new cnt: 1764.0 20.0\n", + "ori cnt and new cnt: 4062.0 259.0\n", + "ori cnt and new cnt: 406.0 7.0\n", + "ori cnt and new cnt: 1872.0 39.0\n", + "ori cnt and new cnt: 645.0 32.0\n", + "ori cnt and new cnt: 2655.0 20.0\n", + "ori cnt and new cnt: 154.0 6.0\n", + "ori cnt and new cnt: 997.0 30.0\n", + "ori cnt and new cnt: 523.0 29.0\n", + "ori cnt and new cnt: 523.0 29.0\n", + "ori cnt and new cnt: 2317.0 95.0\n", + "ori cnt and new cnt: 419.0 10.0\n", + "ori cnt and new cnt: 813.0 13.0\n", + "ori cnt and new cnt: 18796.0 1238.0\n", + "ori cnt and new cnt: 3327.0 113.0\n", + "ori cnt and new cnt: 963.0 29.0\n", + "ori cnt and new cnt: 500.0 21.0\n", + "ori cnt and new cnt: 610.0 22.0\n", + "ori cnt and new cnt: 879.0 18.0\n", + "ori cnt and new cnt: 1681.0 88.0\n", + "ori cnt and new cnt: 654.0 16.0\n", + "ori cnt and new cnt: 62.0 2.0\n", + "ori cnt and new cnt: 1230.0 151.0\n", + "ori cnt and new cnt: 786.0 40.0\n", + "ori cnt and new cnt: 1454.0 22.0\n", + "ori cnt and new cnt: 1237.0 29.0\n", + "ori cnt and new cnt: 1610.0 60.0\n", + "ori cnt and new cnt: 383.0 20.0\n", + "ori cnt and new cnt: 766.0 22.0\n", + "ori cnt and new cnt: 2361.0 20.0\n", + "ori cnt and new cnt: 120.0 3.0\n", + "ori cnt and new cnt: 714.0 31.0\n", + "ori cnt and new cnt: 2185.0 137.0\n", + "ori cnt and new cnt: 1270.0 75.0\n", + "ori cnt and new cnt: 506.0 24.0\n" + ] + } + ], + "source": [ + "# analyze tokens using frequency analysis\n", + "analyze_token_usage(data_root, extended_tokenizer_path, batch_size, keys, token_usage_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "ea495ba4-8b65-4560-94a6-269e8af8a83a", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# path to save selected high-frequency tokens (new tokens to be added)\n", + "high_freq_tokens_path = f\"./models/tokenizer/{model_type}/new_tokenizer/{model_type}_freq_analy_new_token.json\"\n", + "\n", + "# hyperparameter \n", + "freq_threshold = 0.98" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "b816793c-a77c-4cfa-8317-f278cfdbe247", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "./data/all_jsonl_data_sample/7db92aa7a05ae3eb86ec8bd0ab6e6768.lef.gz-0.jsonl\n", + "[4 4 2 2 2 1 1 1 1 1 1 1 1] 21.56\n", + "[4 4 2 2 2 1 1 1 1 1 1 1 1] 21.56\n", + "3\n", + "./data/all_jsonl_data_sample/7d3eb10b8384155f4f262b9d4a9d95b2.lef.gz-0.jsonl\n", + "[4 2 2 2 2 2 1 1 1 1 1 1] 19.6\n", + "[4 2 2 2 2 2 1 1 1 1 1 1] 19.6\n", + "3\n", + "./data/all_jsonl_data_sample/7d1e17d8e8367778544c7664a0dcca34.scala.gz-0.jsonl\n", + "[31 30 30 7 4 3 3 3 3 3 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1] 253.82\n", + "[31 30 30 7 4 3 3 3 3 3 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1] 253.82\n", + "[31 30 30 7 4 3 3 3 3 3 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1] 253.82\n", + "[31 30 30 7 4 3 3 3 3 3 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1] 253.82\n", + "[31 30 30 7 4 3 3 3 3 3 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1] 253.82\n", + "31\n", + "./data/all_jsonl_data_sample/7d9023bf5e97a4417b3e3d15bd0155e5.v.gz-0.jsonl\n", + "[2 1 1 1 1 1] 6.859999999999999\n", + "1\n", + "./data/all_jsonl_data_sample/7d4746f7028947dbbf6f6be1e705a343.h.gz-0.jsonl\n", + "[12 6 5 4 3 2 1 1 1 1 1 1 1] 38.22\n", + "[12 6 5 4 3 2 1 1 1 1 1 1 1] 38.22\n", + "[12 6 5 4 3 2 1 1 1 1 1 1 1] 38.22\n", + "[12 6 5 4 3 2 1 1 1 1 1 1 1] 38.22\n", + "12\n", + "./data/all_jsonl_data_sample/7de4bd2089ed29650c6813a692c0b7fd.cdl.gz-0.jsonl\n", + "[5 5 3 3 3 3 3 2 2 1 1 1] 31.36\n", + "[5 5 3 3 3 3 3 2 2 1 1 1] 31.36\n", + "4\n", + "./data/all_jsonl_data_sample/7da75b519311e22a70fe54061b51b67c.sv.gz-0.jsonl\n", + "[2 2 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1] 19.6\n", + "1\n", + "./data/all_jsonl_data_sample/7d2caac63ccb0ee43f143dfad745a878.v.gz-0.jsonl\n", + "[2 1 1 1 1] 5.88\n", + "1\n", + "./data/all_jsonl_data_sample/7d3ac231744dee023fffc01079a56367.v.gz-0.jsonl\n", + "[4 3 3 3 3 3 2 2 1 1 1 1 1 1 1] 29.4\n", + "[4 3 3 3 3 3 2 2 1 1 1 1 1 1 1] 29.4\n", + "3\n", + "./data/all_jsonl_data_sample/7d223aaa5ad782ba0e026a4fcd6a5e0d.v.gz-0.jsonl\n", + "[4 3 3 3 3 3 2 2 1 1 1 1 1 1] 28.419999999999998\n", + "[4 3 3 3 3 3 2 2 1 1 1 1 1 1] 28.419999999999998\n", + "3\n", + "./data/all_jsonl_data_sample/7d233e7cb17ecddca9baf0704309e739.v.gz-0.jsonl\n", + "[4 3 3 3 3 3 2 2 1 1 1 1 1 1] 28.419999999999998\n", + "[4 3 3 3 3 3 2 2 1 1 1 1 1 1] 28.419999999999998\n", + "3\n", + "./data/all_jsonl_data_sample/7de185d29809f5259616436204ae6c07.spice.gz-0.jsonl\n", + "[21 20 20 16 16 1 1] 93.1\n", + "[21 20 20 16 16 1 1] 93.1\n", + "6\n", + "./data/all_jsonl_data_sample/7d398c165432cac33c33442b2b2b9915.v.gz-0.jsonl\n", + "[3 2 1 1 1 1 1] 9.8\n", + "[3 2 1 1 1 1 1] 9.8\n", + "3\n", + "./data/all_jsonl_data_sample/7db39e3425d097664d5b3aa4800501ad.h.gz-0.jsonl\n", + "[8 1 1 1 1 1] 12.74\n", + "[8 1 1 1 1 1] 12.74\n", + "6\n", + "./data/all_jsonl_data_sample/7dbb099365a3ef31bbc60c3fc37be762.qip.gz-0.jsonl\n", + "[143 27 26 25 20 19 19 17 17 17 17 17 17 17 17 17 16 16\n", + " 13 12 11 11 11 10 10 10 10 9 8 8 8 7 7 7 7 7\n", + " 6 6 6 6 6 6 6 6 6 6 6 5 5 5 5 5 5 5\n", + " 5 5 5 5 5 4 4 4 4 4 4 4 4 4 4 4 4 4\n", + " 4 4 4 4 4 4 4 4 4 4 4 3 3 3 3 3 3 3\n", + " 3 3 3 3 3 3 3 3 3 3 3 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1] 1213.24\n", + "[143 27 26 25 20 19 19 17 17 17 17 17 17 17 17 17 16 16\n", + " 13 12 11 11 11 10 10 10 10 9 8 8 8 7 7 7 7 7\n", + " 6 6 6 6 6 6 6 6 6 6 6 5 5 5 5 5 5 5\n", + " 5 5 5 5 5 4 4 4 4 4 4 4 4 4 4 4 4 4\n", + " 4 4 4 4 4 4 4 4 4 4 4 3 3 3 3 3 3 3\n", + " 3 3 3 3 3 3 3 3 3 3 3 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1] 1213.24\n", + "[143 27 26 25 20 19 19 17 17 17 17 17 17 17 17 17 16 16\n", + " 13 12 11 11 11 10 10 10 10 9 8 8 8 7 7 7 7 7\n", + " 6 6 6 6 6 6 6 6 6 6 6 5 5 5 5 5 5 5\n", + " 5 5 5 5 5 4 4 4 4 4 4 4 4 4 4 4 4 4\n", + " 4 4 4 4 4 4 4 4 4 4 4 3 3 3 3 3 3 3\n", + " 3 3 3 3 3 3 3 3 3 3 3 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1] 1213.24\n", + "[143 27 26 25 20 19 19 17 17 17 17 17 17 17 17 17 16 16\n", + " 13 12 11 11 11 10 10 10 10 9 8 8 8 7 7 7 7 7\n", + " 6 6 6 6 6 6 6 6 6 6 6 5 5 5 5 5 5 5\n", + " 5 5 5 5 5 4 4 4 4 4 4 4 4 4 4 4 4 4\n", + " 4 4 4 4 4 4 4 4 4 4 4 3 3 3 3 3 3 3\n", + " 3 3 3 3 3 3 3 3 3 3 3 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1] 1213.24\n", + "[143 27 26 25 20 19 19 17 17 17 17 17 17 17 17 17 16 16\n", + " 13 12 11 11 11 10 10 10 10 9 8 8 8 7 7 7 7 7\n", + " 6 6 6 6 6 6 6 6 6 6 6 5 5 5 5 5 5 5\n", + " 5 5 5 5 5 4 4 4 4 4 4 4 4 4 4 4 4 4\n", + " 4 4 4 4 4 4 4 4 4 4 4 3 3 3 3 3 3 3\n", + " 3 3 3 3 3 3 3 3 3 3 3 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1] 1213.24\n", + "[143 27 26 25 20 19 19 17 17 17 17 17 17 17 17 17 16 16\n", + " 13 12 11 11 11 10 10 10 10 9 8 8 8 7 7 7 7 7\n", + " 6 6 6 6 6 6 6 6 6 6 6 5 5 5 5 5 5 5\n", + " 5 5 5 5 5 4 4 4 4 4 4 4 4 4 4 4 4 4\n", + " 4 4 4 4 4 4 4 4 4 4 4 3 3 3 3 3 3 3\n", + " 3 3 3 3 3 3 3 3 3 3 3 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1] 1213.24\n", + "[143 27 26 25 20 19 19 17 17 17 17 17 17 17 17 17 16 16\n", + " 13 12 11 11 11 10 10 10 10 9 8 8 8 7 7 7 7 7\n", + " 6 6 6 6 6 6 6 6 6 6 6 5 5 5 5 5 5 5\n", + " 5 5 5 5 5 4 4 4 4 4 4 4 4 4 4 4 4 4\n", + " 4 4 4 4 4 4 4 4 4 4 4 3 3 3 3 3 3 3\n", + " 3 3 3 3 3 3 3 3 3 3 3 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n", + " 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n", + " 1 1 1 1 1 1 1] 1213.24\n", + "142\n", + "./data/all_jsonl_data_sample/7dc1209d13f0e65aab95b30e28fdc7b0.spice.gz-0.jsonl\n", + "[29 24 24 15 11 7 1 1 1] 110.74\n", + "[29 24 24 15 11 7 1 1 1] 110.74\n", + "7\n", + "./data/all_jsonl_data_sample/7dddcc0a609031cc37396af611abd521.v.gz-0.jsonl\n", + "[7 7 7 2 2 2 1 1] 28.419999999999998\n", + "[7 7 7 2 2 2 1 1] 28.419999999999998\n", + "[7 7 7 2 2 2 1 1] 28.419999999999998\n", + "7\n", + "./data/all_jsonl_data_sample/7dbf14d1da77bcf50408a3548fba5443.v.gz-0.jsonl\n", + "[4 3 3 3 2 2 1 1 1 1] 20.58\n", + "[4 3 3 3 2 2 1 1 1 1] 20.58\n", + "3\n", + "./data/all_jsonl_data_sample/7d1585b1aef10fb448a5b5b8fbd0b624.v.gz-0.jsonl\n", + "[3 3 3 2 2 2 2 2 2 1] 21.56\n", + "[3 3 3 2 2 2 2 2 2 1] 21.56\n", + "3\n", + "./data/all_jsonl_data_sample/7d0e56309d51283206ed83074b1ccf76.sv.gz-0.jsonl\n", + "[4 4 1 1 1 1 1 1 1 1 1 1] 17.64\n", + "[4 4 1 1 1 1 1 1 1 1 1 1] 17.64\n", + "3\n", + "./data/all_jsonl_data_sample/7dab8d5649d18c7a9c155b51392a6588.v.gz-0.jsonl\n", + "[29 18 4 4 4 4 4 3 3 3 2 2 2 1 1 1 1 1 1] 86.24\n", + "[29 18 4 4 4 4 4 3 3 3 2 2 2 1 1 1 1 1 1] 86.24\n", + "[29 18 4 4 4 4 4 3 3 3 2 2 2 1 1 1 1 1 1] 86.24\n", + "18\n", + "./data/all_jsonl_data_sample/7dbb8e6199e137dcf3d7085bb0ff9975.v.gz-0.jsonl\n", + "[11 3 1 1] 15.68\n", + "[11 3 1 1] 15.68\n", + "4\n", + "./data/all_jsonl_data_sample/7d62d6bd44676d9f2ea5cdbbd594c4ef.c.gz-0.jsonl\n", + "[2] 1.96\n", + "2\n", + "./data/all_jsonl_data_sample/7dbfcd8e236b3b802c78e9ab57b3a1d0.scala.gz-0.jsonl\n", + "[63 11 7 7 5 4 4 3 3 3 3 3 2 2 2 2 2 2 2 2 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1] 147.98\n", + "[63 11 7 7 5 4 4 3 3 3 3 3 2 2 2 2 2 2 2 2 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1] 147.98\n", + "[63 11 7 7 5 4 4 3 3 3 3 3 2 2 2 2 2 2 2 2 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1] 147.98\n", + "[63 11 7 7 5 4 4 3 3 3 3 3 2 2 2 2 2 2 2 2 1 1 1 1\n", + " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1] 147.98\n", + "36\n", + "./data/all_jsonl_data_sample/7d7b96f51a734da259a6a2ecf379cded.cdl.gz-0.jsonl\n", + "[6 6 4 4 4 4 3 2 2 2 1 1 1] 39.2\n", + "[6 6 4 4 4 4 3 2 2 2 1 1 1] 39.2\n", + "[6 6 4 4 4 4 3 2 2 2 1 1 1] 39.2\n", + "6\n", + "./data/all_jsonl_data_sample/7d23810b472d58f3487c52b1f773189a.lef.gz-0.jsonl\n", + "[4 3 2 2 2 1 1 1 1 1 1 1 1 1] 21.56\n", + "[4 3 2 2 2 1 1 1 1 1 1 1 1 1] 21.56\n", + "3\n", + "./data/all_jsonl_data_sample/7dc70412013409c8b16c0c9e5f14fcfa.v.gz-0.jsonl\n", + "[7 7 7 2 2 2 1 1] 28.419999999999998\n", + "[7 7 7 2 2 2 1 1] 28.419999999999998\n", + "[7 7 7 2 2 2 1 1] 28.419999999999998\n", + "7\n", + "./data/all_jsonl_data_sample/7d5dd4296bf9ada66d63916f69a46faf.emf.gz-0.jsonl\n", + "[20 14 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1] 58.8\n", + "[20 14 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1] 58.8\n", + "[20 14 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1] 58.8\n", + "18\n", + "./data/all_jsonl_data_sample/7dd8f2f49230dee32da3142ac3984412.v.gz-0.jsonl\n", + "[3 2 2 2 2 2 1 1 1 1 1 1 1] 19.6\n", + "[3 2 2 2 2 2 1 1 1 1 1 1 1] 19.6\n", + "3\n", + "./data/all_jsonl_data_sample/7d43a09a7438300752244fb0d9fb05e8.v.gz-0.jsonl\n", + "[3 3 3 2 2 2 2 2 2 1] 21.56\n", + "[3 3 3 2 2 2 2 2 2 1] 21.56\n", + "3\n", + "./data/all_jsonl_data_sample/7de4d2d50bb4424b0fe35bae7a83be7b.lef.gz-0.jsonl\n", + "[4 2 2 2 2 1 1 1 1 1 1 1 1] 19.6\n", + "[4 2 2 2 2 1 1 1 1 1 1 1 1] 19.6\n", + "3\n", + "./data/all_jsonl_data_sample/7d5db7ecf8c09e4d6872e9998d3ffc4c.v.gz-0.jsonl\n", + "[2 1] 2.94\n", + "1\n", + "./data/all_jsonl_data_sample/7d89dd77dcd8779cd013cdad4527558c.h.gz-0.jsonl\n", + "[3 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1] 30.38\n", + "[3 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1] 30.38\n", + "3\n", + "./data/all_jsonl_data_sample/7d5a331f93e5a37c5e367230ca1c1a14.cdl.gz-0.jsonl\n", + "[16 14 14 14 11 11 8 7 7 5 5 4 3 3 2 2 2 2 2 2 1 1 1] 134.26\n", + "[16 14 14 14 11 11 8 7 7 5 5 4 3 3 2 2 2 2 2 2 1 1 1] 134.26\n", + "[16 14 14 14 11 11 8 7 7 5 5 4 3 3 2 2 2 2 2 2 1 1 1] 134.26\n", + "[16 14 14 14 11 11 8 7 7 5 5 4 3 3 2 2 2 2 2 2 1 1 1] 134.26\n", + "15\n", + "./data/all_jsonl_data_sample/7deb467f7b1a328162b5b8ae171ca139.scala.gz-0.jsonl\n", + "[30 7 6 6 6 4 3 3 2 2 2 1 1 1 1] 73.5\n", + "[30 7 6 6 6 4 3 3 2 2 2 1 1 1 1] 73.5\n", + "[30 7 6 6 6 4 3 3 2 2 2 1 1 1 1] 73.5\n", + "[30 7 6 6 6 4 3 3 2 2 2 1 1 1 1] 73.5\n", + "14\n", + "./data/all_jsonl_data_sample/7d146ea4e04027f987bc4c8d1bf2326e.cdl.gz-0.jsonl\n", + "[4 4 3 2 2 2 2 2 1 1 1] 23.52\n", + "[4 4 3 2 2 2 2 2 1 1 1] 23.52\n", + "3\n" + ] + } + ], + "source": [ + "# selecting the top-K tokens in a way that their cumulative frequency accounts for approximately 98%\n", + "get_high_freq_tokens(token_usage_path, high_freq_tokens_path, float(freq_threshold))" + ] + }, + { + "cell_type": "markdown", + "id": "b040dc93-6667-4c8c-91a7-dd715151bbc6", + "metadata": {}, + "source": [ + "## Step 5: Initialize the embeddings of the new tokens by utilizing the extended general-purpose tokenizer" + ] + }, + { + "cell_type": "markdown", + "id": "b2e64e1d-fb92-4cd5-b19a-ff411fb231d7", + "metadata": {}, + "source": [ + "Here we use the `extend_tokenizer` helper fucntion to first add high freq. tokens identified in Step 4 to original tokenizer vocab.​\n", + "\n", + "Both the embedding table and the output layer weights of the original tokenizer depend on the vocab size. Since the vocab size is now changed due to addition of high freq. domain specific tokens, both of these need to be updated.\n", + "\n", + "`extend_sentencepiece` initializes the embeddings of the new tokens by utilizing the general-purpose tokenizer. When a new token (a word or subword unit) is encountered, it is first broken down (tokenized) using the pretrained general-purpose tokenizer. \n", + "\n", + "The new token doesn’t have a predefined embedding (a numerical representation). The embedding of the new token is determined by averaging the embeddings of the tokens generated by the general-purpose tokenizer. For example, if the new token is split into three sub-tokens, the embeddings of these three sub-tokens are averaged to form the embedding of the new token.\n", + "\n", + "Similarly, the weights in the output layer corresponding to the new token are also initialized to the average of the tokens generated by the general-purpose tokenizer. For example, if the new token is split into three sub-tokens, the weights corresponding to these three sub-tokens are averaged to form the weights corresponding to the new token.\n", + "\n", + "Once done, in Step 6 we will merge the new embeddings with the original embedding table (in llama2) to get the final Domain Adapted Tokenizer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2644f313-66d8-47d0-9fc0-f1b2edc72d79", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "ori_tokenizer_path = f\"./models/tokenizer/{model_type}/original_tokenizer/tokenizer.model\" # original sentencepiece tokenizer model\n", + "new_vocab_path = f\"./models/tokenizer/{model_type}/new_tokenizer/freq_vocab.json\" # path to record added new tokens\n", + "old_ebd_path = f\"./models/weight/{model_type}/ori_{model_type}-hf_weight/\" # original embeddings\n", + "new_ebd_path = f\"./models/weight/{model_type}/new_{model_type}-hf_weight/\" # path to store augmented embeddings\n", + "domain_adapter_tokenizer_path = f\"./models/tokenizer/{model_type}/new_tokenizer/tokenizer_freq.model\" # augmented sentencepiece model\n", + "split = 8 # num of partitions to split the augmented embeddings" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3946599e-43b6-4391-b6ab-0068f9f93113", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "f = open(high_freq_tokens_path, \"r\")\n", + "new_tokens = json.load(f)\n", + "print(\"new_tokens: \", new_tokens)\n", + "extend_tokenizer_high_freq_tokens(data_root, ori_tokenizer_path, new_tokens, new_vocab_path, domain_adapter_tokenizer_path, old_ebd_path, new_ebd_path, split)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fda5c5ad", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "print(new_ebd_path) #New weights" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27f8da73", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "print(domain_adapter_tokenizer_path) # domained adapted tokenizer" + ] + }, + { + "cell_type": "markdown", + "id": "227c5b66-bb02-4fc6-96c3-c4284d2f6e99", + "metadata": {}, + "source": [ + "# Step 6: Merge the new embeddings with the original embedding table (in llama2) to get the final Domain Adapted Tokenizer and Embeddings." + ] + }, + { + "cell_type": "markdown", + "id": "05999ff2", + "metadata": {}, + "source": [ + "Helper function `merge_embed` takes the original embeddings downloaded from hugging face, and the augmented embeddings generated in Step 5 above, merges them and then saves the result at `save_path`.\n", + "\n", + "For instance, figure below shows an illustration of embedding table modification. Here each row corresponds to a unique token and each column represents a dimension of the embedding vector. The size of the vocabulary determines the number of rows in the embedding table. The embedding layer in the LLM which is responsible for converting the data into numerical vectors uses the embedding table to perform this conversion. The dimensionality of the embedding layer is given by the number of columns in the embedding table.
\n", + "\n", + "![pipeline](imgs/embedding_table.png)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21658fb2-a54a-41cf-94fc-81735767cdab", + "metadata": {}, + "outputs": [], + "source": [ + "os.makedirs(f\"/models/weight/new_merged_{model_type}-hf\", exist_ok=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4c616e7f", + "metadata": { + "scrolled": true, + "tags": [] + }, + "outputs": [], + "source": [ + "old_ebd_path = f\"./models/weight/{model_type}-hf\" # original embeddings downloaded from hf\n", + "new_ebd_path = f\"./models/weight/{model_type}/new_{model_type}-hf_weight\" # augmented embeddings\n", + "save_path = f\"./models/weight/new_merged_{model_type}-hf\" # Path to adapted llama2 weights\n", + "merge_embed(old_ebd_path, new_ebd_path, save_path)" + ] + }, + { + "cell_type": "markdown", + "id": "9060868f", + "metadata": {}, + "source": [ + "### New weights and tokenizer are stored at:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ca159e0c-28b2-4a64-bdf4-314e4191c2a0", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "print(new_ebd_path) #New weights" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9d657312-814a-4ad8-a46f-c9c7bc4ee978", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "print(domain_adapter_tokenizer_path) # domained adapted tokenizer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d0782e9c", + "metadata": { + "scrolled": true, + "tags": [] + }, + "outputs": [], + "source": [ + "# check layers and dimensions (optional)\n", + "state_dict = torch.load(f'{save_path}/consolidated.01.pth')\n", + "for index, (key, value) in enumerate(state_dict.items()):\n", + " print(f\"Index: {index}, layer: {key}, Layer size: {value.size()}\")" + ] + }, + { + "cell_type": "markdown", + "id": "15a172e2", + "metadata": {}, + "source": [ + "# Next Step\n", + "\n", + "The final Domain adapted Tokenizer obtained using this notebook can be used in a continual pre-training pipeline for domain adaptive pretraining." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tutorials/llm/llama/domain-adaptive-pretraining/code/domain_adaptive_pretraining_nemo1.0.ipynb b/tutorials/llm/llama/domain-adaptive-pretraining/code/domain_adaptive_pretraining_nemo1.0.ipynb new file mode 100644 index 000000000000..3d48fd6e8b24 --- /dev/null +++ b/tutorials/llm/llama/domain-adaptive-pretraining/code/domain_adaptive_pretraining_nemo1.0.ipynb @@ -0,0 +1,489 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cd13460c", + "metadata": {}, + "source": [ + "# Domain Adaptive Pre-Training (DAPT)\n", + "\n", + "## Goal\n", + "\n", + "Given a foundational language model (in this case llama-2-7B) that was pre-trained on a broad, general-purpose corpus, our goal is to further pretrain the model on a specific domain (in this example, ChipDesign) to enhance its understanding of domain-specific language and context. This process is called Domain-Adaptive Pretraining (DAPT). DAPT adapts a general-purpose model to specialized tasks within a particular field. Instead of training from scratch, we aim to “specialize” the model by focusing on a target domain corpus, allowing it to adapt to the unique vocabulary, semantics, and syntax of that field.\n", + "\n", + "Our primary goals with respect to DAPT are as follows:\n", + "* Improve the model’s performance and accuracy on domain-specific tasks\n", + "* Ensure the model retains general language capabilities\n", + "* Minimize pretraining time by leveraging existing knowledge in the model\n", + "\n", + "DAPT typically enhances a model’s efficacy in downstream tasks for the domain by exposing it to domain-relevant texts. This pretraining phase can result in more accurate and context-aware predictions on domain-specific data, as the model gains an understanding of field-specific terminology, abbreviations, and common phrases." + ] + }, + { + "cell_type": "markdown", + "id": "c43ef563", + "metadata": {}, + "source": [ + "# NeMo Tools and Resources\n", + "\n", + "* [NeMo Framework](https://docs.nvidia.com/nemo-framework/user-guide/latest/overview.html)" + ] + }, + { + "cell_type": "markdown", + "id": "bea0b51f", + "metadata": {}, + "source": [ + "# Software Requirements\n", + "* Access to latest NeMo Framework NGC Containers\n", + "* This playbook has been tested on: nvcr.io/nvidia/nemo:dev. It is expected to work similarly on other environments.\n", + "\n", + "\n", + "#### Launch the NeMo Framework container as follows: \n", + "\n", + "```\n", + "docker run -it -p 8080:8080 -p 8088:8088 --rm --gpus '\"device=0,1\"' --ipc=host --network host -v $(pwd):/workspace nvcr.io/nvidia/nemo:dev\n", + "```\n", + "\n", + "#### Launch Jupyter Notebook as follows: \n", + "```\n", + "jupyter notebook --allow-root --ip 0.0.0.0 --port 8088 --no-browser --NotebookApp.token=''\n", + "\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "id": "7137e1db", + "metadata": {}, + "source": [ + "# Hardware Requirements\n", + "\n", + "* This playbook has been tested on 2xA100 80G but can be scaled to multiple GPUs as well as multiple nodes by modifying the appropriate parameters" + ] + }, + { + "cell_type": "markdown", + "id": "91ecb0d3", + "metadata": {}, + "source": [ + "# Data\n", + "\n", + "* In this playbook, we will leverage chip domain/hardware datasets from open-source GitHub repositories, wiki URLs, and academic papers. Data has been processed and curated using [NeMo Curator](https://github.com/NVIDIA/NeMo-Curator/tree/main) as shown in this [playbook](https://github.com/jvamaraju/ndc_dapt_playbook/tree/dapt_jv)" + ] + }, + { + "cell_type": "markdown", + "id": "ba16a72b", + "metadata": {}, + "source": [ + "# Notebook Outline\n", + "\n", + "* Step 1: Prepare the data for pretraining. This is a multi-step process discussed in detail later in the specific section (later in the notebook).\n", + "\n", + "* Step 2: Download the llama-2-7B hugging face checkpoint and convert to .nemo format.\n", + "\n", + "* Step 3: Continued pretraining the llama-2-7b model using the prepared data and the custom trained tokenizer (from the previous notebook)." + ] + }, + { + "cell_type": "markdown", + "id": "ec372453", + "metadata": {}, + "source": [ + "# Step 1: Data Preparation for pretraining\n", + "\n", + "Identify the different file types (example: code, text, etc) in the pretraining data, in this case we only have 'code' type files. This is typically dataset dependent. \n", + "\n", + "If you used the Data Curation tutorial as instructed in the Readme, you can point ```data_path ``` variable to the path containing the curated data." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "2c935b99", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Number of Files containing 'file_type':'text': 0\n", + "Number of Files containing 'file_type':'code': 8835\n" + ] + } + ], + "source": [ + "import os\n", + "import json\n", + "\n", + "# Function to count the number of files in each of the different file types- code, text\n", + "def identify_jsonl_files(data_path):\n", + " code_files = []\n", + " text_files = []\n", + " cnt_text = 0\n", + " cnt_code = 0\n", + " for root, _, files in os.walk(data_path):\n", + " for file in files:\n", + " if file.endswith('.jsonl'):\n", + " file_path = os.path.join(root, file)\n", + " with open(file_path, 'r') as f:\n", + " has_code = False\n", + " has_text = False\n", + " for line in f:\n", + " try:\n", + " json_obj = json.loads(line.strip())\n", + " file_type = json_obj.get('file_type', '').lower()\n", + " if file_type == 'code':\n", + " has_code = True\n", + " elif file_type == 'text':\n", + " has_text = True\n", + " if has_code and has_text:\n", + " break\n", + " except json.JSONDecodeError:\n", + " continue\n", + " if has_code:\n", + " code_files.append(file_path)\n", + " cnt_code = cnt_code + 1\n", + " if has_text:\n", + " text_files.append(file_path)\n", + " cnt_text = cnt_text + 1\n", + " return code_files, text_files, cnt_code, cnt_text\n", + "\n", + "# Modify data path to point to jsonl data source, in this case data_path='code/data/all_jsonl_data'\n", + "data_path = 'code/data/all_jsonl_data'\n", + "\n", + "code_files, text_files, cnt_code, cnt_text = identify_jsonl_files(data_path)\n", + "\n", + "print(\"\\nNumber of Files containing 'file_type':'text':\", cnt_text)\n", + "print(\"Number of Files containing 'file_type':'code':\", cnt_code)" + ] + }, + { + "cell_type": "markdown", + "id": "60987ff2", + "metadata": {}, + "source": [ + "### Merging code JSONL files into a single JSONL file for further preprocessing" + ] + }, + { + "cell_type": "markdown", + "id": "c02f2e6f", + "metadata": {}, + "source": [ + "This is an optional step, it is possible to use multiple jsonl files in this workflow as well. This example uses a single merged. jsonl file" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "892f4493", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import json\n", + "\n", + "def list_jsonl_files(directory):\n", + " jsonl_files = []\n", + " for root, _, files in os.walk(directory):\n", + " for file in files:\n", + " if file.endswith('.jsonl'):\n", + " jsonl_files.append(os.path.join(root, file))\n", + " return jsonl_files\n", + "\n", + "# Function to merge multiple jsonl files into a single file \n", + "def merge_jsonl_files(directory, output_file):\n", + " jsonl_files = list_jsonl_files(directory)\n", + " \n", + " with open(output_file, 'w') as outfile:\n", + " for input_file in jsonl_files:\n", + " with open(input_file, 'r') as infile:\n", + " for line in infile:\n", + " try:\n", + " json_object = json.loads(line.strip())\n", + " json.dump(json_object, outfile)\n", + " outfile.write('\\n')\n", + " except json.JSONDecodeError:\n", + " print(f\"Skipping invalid JSON in {input_file}: {line.strip()}\")\n", + "\n", + " print(f\"Merged {len(jsonl_files)} JSONL files into {output_file}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9bb0c80a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Merged 8835 JSONL files into code_merged_output.jsonl\n" + ] + } + ], + "source": [ + "directory = 'code/data/all_jsonl_data'\n", + "output_file = 'code_merged_output.jsonl'\n", + "merge_jsonl_files(directory, output_file)" + ] + }, + { + "cell_type": "markdown", + "id": "6d00ad63", + "metadata": {}, + "source": [ + "### Data Format Conversion for pretraining: JSONL to bin/idx files \n", + "\n", + "For efficient pretraining, we convert data from JSONL to bin/idx format. \n", + "\n", + "JSONL files, while convenient for storing structured text data, are not optimized for high-speed data loading during large language model training. In pretraining workflows, particularly those with large datasets and complex model architectures, the need for fast data access and efficient memory management is essential.\n", + "\n", + "The bin/idx format is a binary format specifically designed to facilitate high-throughput data loading. This format allows direct, randomized access to data samples, which speeds up I/O operations and reduces the memory footprint compared to loading JSONL files. By converting data to bin/idx format, hardware utilization can be maximized and bottlenecks in data processing can be avoided, leading to a more efficient pretraining process.\n", + "\n", + "#### Benefits of bin/idx format for Pretraining:\n", + "\n", + "* **Optimized I/O Performance:** The binary format enables quicker data reads and reduces latency, allowing the model to continuously access data at high speeds.\n", + "* **Efficient Memory Usage:** Data in bin/idx format consumes less memory during loading, making it suitable for large datasets and enabling better use of available system resources.\n", + "* **Enhanced Scalability:** With bin/idx, it’s easier to handle shuffling and batching of large datasets, which is essential for pretraining on diverse domain-specific data." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "709f2c08", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "merges.txt\t\t tokenizer.json\t\tvocab.json\r\n", + "special_tokens_map.json tokenizer_config.json\r\n" + ] + } + ], + "source": [ + "# After the running through the custom_tokenization.ipynb, you would have \n", + "# the new domain adpated tokenizer model in the following directory\n", + "!ls models/tokenizer/llama2/custom_tokenizer_init_20000_json" + ] + }, + { + "cell_type": "markdown", + "id": "de696d7b", + "metadata": {}, + "source": [ + "Modify the `input` to point to the merged `jsonl` file. Similarly modify paths to `vocab`, `tokenizer-model`, `merge-file` to point to relevant file paths. \n", + "\n", + "In the following code block, ```tokenizer-model``` is set to using the original tokenizer that comes as a part of llama2-7b-hf, but `tokenizer-model` should point to the custom tokenizer (trained in the custom tokenizer training notebook) if your data has domain specific terminology" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dcbf66a2", + "metadata": {}, + "outputs": [], + "source": [ + "!python3 /opt/NeMo/scripts/nlp_language_modeling/preprocess_data_for_megatron.py \\\n", + "--input='code_merged_output.jsonl' \\\n", + "--json-keys=text \\\n", + "--tokenizer-library=sentencepiece \\\n", + "--vocab 'models/tokenizer/llama2/custom_tokenizer_init_20000_json/vocab.json' \\\n", + "--dataset-impl mmap \\\n", + "--tokenizer-model '/workspace/Llama-2-7b-hf/tokenizer.model' \\\n", + "--tokenizer-type llama \\\n", + "--merge-file 'models/tokenizer/llama2/custom_tokenizer_init_20000_json/merges.txt' \\\n", + "--append-eod \\\n", + "--output-prefix='preprocessed_data'" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "0f05efa5", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "README.md\t\t\t nemo_experiments\r\n", + "cdeng\t\t\t\t preprocessed_data_text_document\r\n", + "code\t\t\t\t preprocessed_data_text_document.bin\r\n", + "code_merged_output.jsonl\t preprocessed_data_text_document.idx\r\n", + "domain_adaptive_pretraining.ipynb venv\r\n" + ] + } + ], + "source": [ + "# If the above step runs successfully, two files with the extensions .bin and .idx will be generated\n", + "!ls " + ] + }, + { + "cell_type": "markdown", + "id": "82f95149", + "metadata": {}, + "source": [ + "# Step 2: Download Llama-2-7b Hugging Face checkpoint and convert to .nemo checkpoint\n", + "\n", + "The code below assumes you already have the llama-2-7b checkpoint downloaded in ```/workspace/Llama-2-7b-hf/```\n", + "\n", + "Llama-2-7b-hf checkpoint can be downloaded from https://huggingface.co/meta-llama/Llama-2-7b-hf/tree/main" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "46c7f997", + "metadata": {}, + "outputs": [], + "source": [ + "!python /opt/NeMo/scripts/checkpoint_converters/convert_llama_hf_to_nemo.py --input_name_or_path=/workspace/Llama-2-7b-hf/ --output_path=/workspace/llama2-7b.nemo" + ] + }, + { + "cell_type": "markdown", + "id": "b94e774b", + "metadata": {}, + "source": [ + "The conversion will generate a ```llama2-7b.nemo``` file which can be used for the continued pretraining using NeMo Toolkit as shown in Step 3. " + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "c689e584", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Llama-2-7b-hf\t\t dapt-custom-tokenization megatron_llama\r\n", + "bin-idx-conversion.ipynb dapt-data-curation\t megatron_llama_config.yaml\r\n", + "convert.py\t\t llama2-7b.nemo\t sentencepiece\r\n", + "custom-tokenizer\t loader_llama2.py\t venv\r\n" + ] + } + ], + "source": [ + "!ls /workspace" + ] + }, + { + "cell_type": "markdown", + "id": "fe1bdfe0", + "metadata": {}, + "source": [ + "# Step 3: Continued Pretraining using Llama2-7b with NeMo\n", + "\n", + "For this step `megatron_gpt_pretraining.py` from NeMo Toolkit is used for continued pretraining, this step allows to configure different parameters for the pretraining depending on the set up. For example `trainer.devices` `model.tensor_model_parallel_size` depend on the number of GPUs available for this job. \n", + "\n", + "Additionally, specify the path to the custom trained tokenizer for `model.tokenizer.model`, the `.nemo` checkpoint for `model.restore_from_path`. \n", + "\n", + "The `model.data.data_prefix` is specified in the form [weightage to data, datafile] Example `[1,preprocessed_data_text_document]` assigns the whole weightage [=1] to `preprocessed_data_text_document`. If there are multiple files, different weightage (should sum to 1) can be assigned to each file to control the data blend for pretraining. \n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a40f547", + "metadata": {}, + "outputs": [], + "source": [ + "# Test out the pretraining set up with mock data: model.data.data_impl=mock\n", + "\n", + "!python /opt/NeMo/examples/nlp/language_modeling/megatron_gpt_pretraining.py \\\n", + " --config-path=/opt/NeMo/examples/nlp/language_modeling/conf \\\n", + " --config-name=megatron_llama_config \\\n", + " trainer.precision=bf16 \\\n", + " trainer.devices=1 \\\n", + " trainer.num_nodes=1 \\\n", + " trainer.max_steps=2 \\\n", + " trainer.val_check_interval=8 \\\n", + " model.data.data_impl=mock \\\n", + " model.micro_batch_size=1 \\\n", + " model.global_batch_size=4 \\\n", + " model.tensor_model_parallel_size=1 \\\n", + " model.pipeline_model_parallel_size=1 \\\n", + " model.tokenizer.library=sentencepiece \\\n", + " model.tokenizer.model=/workspace/Llama-2-7b-hf/tokenizer.model \\\n", + " +model.restore_from_path=/workspace/llama2-7b.nemo \\\n", + " exp_manager.name=megatron_llama_continual \\\n", + " exp_manager.resume_ignore_no_checkpoint=false \\\n", + " exp_manager.resume_if_exists=false " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "71672ff4", + "metadata": {}, + "outputs": [], + "source": [ + "# Pretraining using preprocessed data (+model.data.data_prefix)\n", + "\n", + "!python /opt/NeMo/examples/nlp/language_modeling/megatron_gpt_pretraining.py \\\n", + " --config-path=/opt/NeMo/examples/nlp/language_modeling/conf \\\n", + " --config-name=megatron_llama_config \\\n", + " trainer.precision=bf16 \\\n", + " trainer.devices=2 \\\n", + " trainer.num_nodes=1 \\\n", + " trainer.max_steps=5 \\\n", + " trainer.val_check_interval=8 \\\n", + " model.micro_batch_size=1 \\\n", + " model.global_batch_size=4 \\\n", + " model.tensor_model_parallel_size=2 \\\n", + " model.pipeline_model_parallel_size=1 \\\n", + " model.tokenizer.library=sentencepiece \\\n", + " model.tokenizer.model=/workspace/Llama-2-7b-hf/tokenizer.model \\\n", + " model.megatron_amp_O2=True \\\n", + " +model.restore_from_path=/workspace/llama2-7b.nemo \\\n", + " +model.data.data_prefix=[1,preprocessed_data_text_document] \\\n", + " exp_manager.name=megatron_llama_continual \\\n", + " exp_manager.resume_ignore_no_checkpoint=true \\\n", + " exp_manager.resume_if_exists=false " + ] + }, + { + "cell_type": "markdown", + "id": "cf30d8c8", + "metadata": {}, + "source": [ + "### To monitor the training, launch Tensorboard from another terminal\n", + "\n", + "`tensorboard --logdir nemo_experiments --bind_all`" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tutorials/llm/llama/domain-adaptive-pretraining/code/domain_adaptive_pretraining_nemo2.0.ipynb b/tutorials/llm/llama/domain-adaptive-pretraining/code/domain_adaptive_pretraining_nemo2.0.ipynb new file mode 100644 index 000000000000..c6ca48697fa9 --- /dev/null +++ b/tutorials/llm/llama/domain-adaptive-pretraining/code/domain_adaptive_pretraining_nemo2.0.ipynb @@ -0,0 +1,553 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cd13460c", + "metadata": {}, + "source": [ + "# Domain Adaptive Pre-Training (DAPT)\n", + "\n", + "## Goal\n", + "\n", + "Given a foundational language model (in this case llama-2-7B) that was pre-trained on a broad, general-purpose corpus, our goal is to further pretrain the model on a specific domain (in this example, ChipDesign) to enhance its understanding of domain-specific language and context. This process is called Domain-Adaptive Pretraining (DAPT). DAPT adapts a general-purpose model to specialized tasks within a particular field. Instead of training from scratch, we aim to “specialize” the model by focusing on a target domain corpus, allowing it to adapt to the unique vocabulary, semantics, and syntax of that field.\n", + "\n", + "Our primary goals with respect to DAPT are as follows:\n", + "* Improve the model’s performance and accuracy on domain-specific tasks\n", + "* Ensure the model retains general language capabilities\n", + "* Minimize pretraining time by leveraging existing knowledge in the model\n", + "\n", + "DAPT typically enhances a model’s efficacy in downstream tasks for the domain by exposing it to domain-relevant texts. This pretraining phase can result in more accurate and context-aware predictions on domain-specific data, as the model gains an understanding of field-specific terminology, abbreviations, and common phrases." + ] + }, + { + "cell_type": "markdown", + "id": "c43ef563", + "metadata": {}, + "source": [ + "# NeMo Tools and Resources\n", + "\n", + "* [NeMo Framework](https://docs.nvidia.com/nemo-framework/user-guide/latest/overview.html)" + ] + }, + { + "cell_type": "markdown", + "id": "bea0b51f", + "metadata": {}, + "source": [ + "# Software Requirements\n", + "* Access to latest NeMo Framework NGC Containers\n", + "* This playbook has been tested on: nvcr.io/nvidia/nemo:dev. It is expected to work similarly on other environments.\n", + "\n", + "\n", + "#### Launch the NeMo Framework container as follows: \n", + "\n", + "```\n", + "docker run -it -p 8080:8080 -p 8088:8088 --rm --gpus '\"device=0,1\"' --ipc=host --network host -v $(pwd):/workspace nvcr.io/nvidia/nemo:dev\n", + "```\n", + "\n", + "#### Launch Jupyter Notebook as follows: \n", + "```\n", + "jupyter notebook --allow-root --ip 0.0.0.0 --port 8088 --no-browser --NotebookApp.token=''\n", + "\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "id": "7137e1db", + "metadata": {}, + "source": [ + "# Hardware Requirements\n", + "\n", + "* This playbook has been tested on 2xA100 80G but can be scaled to multiple GPUs as well as multiple nodes by modifying the appropriate parameters" + ] + }, + { + "cell_type": "markdown", + "id": "91ecb0d3", + "metadata": {}, + "source": [ + "# Data\n", + "\n", + "* In this playbook, we will leverage chip domain/hardware datasets from open-source GitHub repositories, wiki URLs, and academic papers. Data has been processed and curated using [NeMo Curator](https://github.com/NVIDIA/NeMo-Curator/tree/main) as shown in this [playbook](https://github.com/jvamaraju/ndc_dapt_playbook/tree/dapt_jv)" + ] + }, + { + "cell_type": "markdown", + "id": "ba16a72b", + "metadata": {}, + "source": [ + "# Notebook Outline\n", + "\n", + "* Step 1: Prepare the data for pretraining. This is a multi-step process discussed in detail later in the specific section (later in the notebook).\n", + "\n", + "* Step 2: Download the llama-2-7B hugging face checkpoint and convert to .nemo format.\n", + "\n", + "* Step 3: Continued pretraining the llama-2-7b model using the prepared data and the custom trained tokenizer (from the previous notebook)." + ] + }, + { + "cell_type": "markdown", + "id": "ec372453", + "metadata": {}, + "source": [ + "# Step 1: Data Preparation for pretraining\n", + "\n", + "Identify the different file types (example: code, text, etc) in the pretraining data, in this case we only have 'code' type files. This is typically dataset dependent. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2c935b99", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import json\n", + "\n", + "from nemo.collections.llm import Llama2Config7B\n", + "\n", + "\n", + "# Function to count the number of files in each of the different file types- code, text\n", + "def identify_jsonl_files(data_path):\n", + " code_files = []\n", + " text_files = []\n", + " cnt_text = 0\n", + " cnt_code = 0\n", + " for root, _, files in os.walk(data_path):\n", + " for file in files:\n", + " if file.endswith('.jsonl'):\n", + " file_path = os.path.join(root, file)\n", + " with open(file_path, 'r') as f:\n", + " has_code = False\n", + " has_text = False\n", + " for line in f:\n", + " try:\n", + " json_obj = json.loads(line.strip())\n", + " file_type = json_obj.get('file_type', '').lower()\n", + " if file_type == 'code':\n", + " has_code = True\n", + " elif file_type == 'text':\n", + " has_text = True\n", + " if has_code and has_text:\n", + " break\n", + " except json.JSONDecodeError:\n", + " continue\n", + " if has_code:\n", + " code_files.append(file_path)\n", + " cnt_code = cnt_code + 1\n", + " if has_text:\n", + " text_files.append(file_path)\n", + " cnt_text = cnt_text + 1\n", + " return code_files, text_files, cnt_code, cnt_text\n", + "\n", + "# Modify data path to point to jsonl data source, in this case data_path='code/data/all_jsonl_data'\n", + "data_path = '/workspace/dapt-custom-tokenization/code/data/all_jsonl_data'\n", + "\n", + "code_files, text_files, cnt_code, cnt_text = identify_jsonl_files(data_path)\n", + "\n", + "print(\"\\nNumber of Files containing 'file_type':'text':\", cnt_text)\n", + "print(\"Number of Files containing 'file_type':'code':\", cnt_code)" + ] + }, + { + "cell_type": "markdown", + "id": "60987ff2", + "metadata": {}, + "source": [ + "### Merging code JSONL files into a single JSONL file for further preprocessing" + ] + }, + { + "cell_type": "markdown", + "id": "c02f2e6f", + "metadata": {}, + "source": [ + "This is an optional step, it is possible to use multiple jsonl files in this workflow as well. This example uses a single merged. jsonl file" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "892f4493", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import json\n", + "\n", + "def list_jsonl_files(directory):\n", + " jsonl_files = []\n", + " for root, _, files in os.walk(directory):\n", + " for file in files:\n", + " if file.endswith('.jsonl'):\n", + " jsonl_files.append(os.path.join(root, file))\n", + " return jsonl_files\n", + "\n", + "# Function to merge multiple jsonl files into a single file \n", + "def merge_jsonl_files(directory, output_file):\n", + " jsonl_files = list_jsonl_files(directory)\n", + " \n", + " with open(output_file, 'w') as outfile:\n", + " for input_file in jsonl_files:\n", + " with open(input_file, 'r') as infile:\n", + " for line in infile:\n", + " try:\n", + " json_object = json.loads(line.strip())\n", + " json.dump(json_object, outfile)\n", + " outfile.write('\\n')\n", + " except json.JSONDecodeError:\n", + " print(f\"Skipping invalid JSON in {input_file}: {line.strip()}\")\n", + "\n", + " print(f\"Merged {len(jsonl_files)} JSONL files into {output_file}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9bb0c80a", + "metadata": {}, + "outputs": [], + "source": [ + "directory = '/workspace/dapt-custom-tokenization/code/data/all_jsonl_data'\n", + "output_file = '/workspace/dapt-custom-tokenization/code_merged_output.jsonl'\n", + "merge_jsonl_files(directory, output_file)" + ] + }, + { + "cell_type": "markdown", + "id": "6d00ad63", + "metadata": {}, + "source": [ + "### Data Format Conversion for pretraining: JSONL to bin/idx files \n", + "\n", + "For efficient pretraining, we convert data from JSONL to bin/idx format. \n", + "\n", + "JSONL files, while convenient for storing structured text data, are not optimized for high-speed data loading during large language model training. In pretraining workflows, particularly those with large datasets and complex model architectures, the need for fast data access and efficient memory management is essential.\n", + "\n", + "The bin/idx format is a binary format specifically designed to facilitate high-throughput data loading. This format allows direct, randomized access to data samples, which speeds up I/O operations and reduces the memory footprint compared to loading JSONL files. By converting data to bin/idx format, hardware utilization can be maximized and bottlenecks in data processing can be avoided, leading to a more efficient pretraining process.\n", + "\n", + "#### Benefits of bin/idx format for Pretraining:\n", + "\n", + "* **Optimized I/O Performance:** The binary format enables quicker data reads and reduces latency, allowing the model to continuously access data at high speeds.\n", + "* **Efficient Memory Usage:** Data in bin/idx format consumes less memory during loading, making it suitable for large datasets and enabling better use of available system resources.\n", + "* **Enhanced Scalability:** With bin/idx, it’s easier to handle shuffling and batching of large datasets, which is essential for pretraining on diverse domain-specific data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "709f2c08", + "metadata": {}, + "outputs": [], + "source": [ + "!ls /workspace/dapt-custom-tokenization/code/code/models/tokenizer/llama2/custom_tokenizer_init_20000.json" + ] + }, + { + "cell_type": "markdown", + "id": "de696d7b", + "metadata": {}, + "source": [ + "Modify the `input` to point to the merged `jsonl` file. Similarly modify paths to `vocab`, `tokenizer-model`, `merge-file` to point to relevant file paths. `tokenizer-model` should point to the custom tokenizer (trained in the custom tokenizer training notebook) if your data has domain specific terminology" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dcbf66a2", + "metadata": {}, + "outputs": [], + "source": [ + "#### Uncomment to use custom trained tokenizer ####\n", + "# !python3 /opt/NeMo/scripts/nlp_language_modeling/preprocess_data_for_megatron.py \\\n", + "# --input='/workspace/dapt-custom-tokenization/code_merged_output.jsonl' \\\n", + "# --json-keys=text \\\n", + "# --tokenizer-library=sentencepiece \\\n", + "# --vocab '/workspace/dapt-custom-tokenization/code/code/models/tokenizer/llama2/custom_tokenizer_init_20000.json/vocab.json' \\\n", + "# --dataset-impl mmap \\\n", + "# --tokenizer-model '/workspace/Llama-2-7b-hf/tokenizer.model' \\\n", + "# --tokenizer-type llama \\\n", + "# --merge-file '/workspace/dapt-custom-tokenization/code/code/models/tokenizer/llama2/custom_tokenizer_init_20000.json/merges.txt' \\\n", + "# --append-eod \\\n", + "# --output-prefix='preprocessed_data'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "89b9583d-1dac-4717-b028-c78d0d703f45", + "metadata": {}, + "outputs": [], + "source": [ + "# Using default Llama-2 tokenizer for testing purpose\n", + "!python3 /opt/NeMo/scripts/nlp_language_modeling/preprocess_data_for_megatron.py \\\n", + "--input='/workspace/dapt-custom-tokenization/code_merged_output.jsonl' \\\n", + "--json-keys=text \\\n", + "--tokenizer-library=sentencepiece \\\n", + "--dataset-impl mmap \\\n", + "--tokenizer-model '/workspace/Llama-2-7b-hf/tokenizer.model' \\\n", + "--tokenizer-type llama \\\n", + "--append-eod \\\n", + "--output-prefix='preprocessed_data'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0f05efa5", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# If the above step runs successfully, two files with the extensions .bin and .idx will be generated\n", + "!ls " + ] + }, + { + "cell_type": "markdown", + "id": "82f95149", + "metadata": {}, + "source": [ + "# Step 2: Download and Import Llama-2-7b Hugging Face checkpoint\n", + "\n", + "Llama2-7B model can be automatically downloaded and converted to NeMo2 format with the following script:\n", + "\n", + "* Save the following code snippet as ```converttonemo2.py```\n", + "* Run ```python3 converttonemo2.py```" + ] + }, + { + "cell_type": "markdown", + "id": "b3260b62-c179-4bc6-b256-729ff6403fa4", + "metadata": {}, + "source": [ + "```\n", + "from nemo.collections import llm\n", + "from nemo.collections.llm import Llama2Config7B\n", + "\n", + "if __name__ == \"__main__\":\n", + " output = llm.import_ckpt(\n", + " model=llm.LlamaModel(config=Llama2Config7B()),\n", + " source=\"hf:///workspace/Llama-2-7b-hf\",\n", + " )\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "46c7f997", + "metadata": {}, + "outputs": [], + "source": [ + "from nemo.collections import llm\n", + "from nemo.collections.llm import Llama2Config7B\n", + "\n", + "if __name__ == \"__main__\":\n", + " output = llm.import_ckpt(\n", + " model=llm.LlamaModel(config=Llama2Config7B()),\n", + " source=\"hf:///workspace/Llama-2-7b-hf\",\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "b94e774b", + "metadata": {}, + "source": [ + "The conversion will generate a ```llama-2-7b.nemo``` which can be used for the continued pretraining using NeMo Toolkit as shown in Step 3 in default ```$NEMO_HOME``` folder." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c689e584", + "metadata": {}, + "outputs": [], + "source": [ + "!ls /workspace" + ] + }, + { + "cell_type": "markdown", + "id": "fe1bdfe0", + "metadata": {}, + "source": [ + "# Step 3: Continued Pretraining using Llama2-7b with NeMo2\n", + "\n", + "For this step we use a predefined recipe `llama2_7b.pretrain_recipe` from NeMo Toolkit for continued pretraining. We will modify the `pretrain_recipe` and use it for continued pretraining workflow. Typically this involves changing dataset files and data blends, changing learning rate scheduler, changing default parallelism based on number of devices available, adding connector to resume training, etc.\n", + "\n", + "First, we define the recipe and executor for using NeMo2 as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a40f547", + "metadata": {}, + "outputs": [], + "source": [ + "import nemo_run as run\n", + "from nemo.collections import llm\n", + "\n", + "# Configure recipe to pre-train based on the default llama-2-7b recipe\n", + "def configure_recipe(nodes: int = 1, gpus_per_node: int = 1):\n", + " recipe = llm.llama2_7b.pretrain_recipe(\n", + " name=\"llama2_7b_pretraining\",\n", + " # Modify based on number of nodes available\n", + " num_nodes=nodes,\n", + " num_gpus_per_node=gpus_per_node,\n", + " )\n", + " # Modify\n", + " recipe.trainer.strategy.context_parallel_size = 1\n", + " recipe.trainer.strategy.tensor_model_parallel_size=1\n", + " recipe.trainer.val_check_interval = 100\n", + " return recipe\n", + "\n", + "# Executor for running pretraining \n", + "def local_executor_torchrun(nodes: int = 1, devices: int = 1) -> run.LocalExecutor:\n", + " # Env vars for jobs are configured here\n", + " # env_vars = {\n", + " # \"TORCH_NCCL_AVOID_RECORD_STREAMS\": \"1\",\n", + " # \"NCCL_NVLS_ENABLE\": \"0\",\n", + " # \"NVTE_DP_AMAX_REDUCE_INTERVAL\": \"0\",\n", + " # \"NVTE_ASYNC_AMAX_REDUCTION\": \"1\",\n", + " # \"NVTE_FUSED_ATTN\": \"0\",\n", + " # }\n", + " # executor = run.LocalExecutor(ntasks_per_node=devices, launcher=\"torchrun\", env_vars=env_vars)\n", + " executor = run.LocalExecutor(ntasks_per_node=devices, launcher=\"torchrun\")\n", + " return executor" + ] + }, + { + "cell_type": "markdown", + "id": "464d303fc973333d", + "metadata": {}, + "source": [ + "Let's instantiate the `recipe` and modify it so that it uses the desired number of GPUs, resuming from the pretrained Llama2-7b checkpoint instead of training from scratch.\n", + "\n", + "The default `recipe` initializes all the essential components required for Llama2 7B pretraining, including model, dataloader, trainer, logger, optimizer etc. `recipe` is not executed during instantiation, so it is very simple to modify it to fit your custom training workflow. In our case, we want to do the DAPT (instead of pretraining from scratch), and all we need to do is to add a `resume` config which points to the Llama2 7B checkpoint.\n", + "\n", + "You can easily change the optimizer, parallelism, data as per your use case. Look at the following example for guidance on how to tweak these parameters. Note: you are only configuring your task at this stage; the underlying code is not executed unless you launch the job using the executor." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b70481ad7579de7a", + "metadata": {}, + "outputs": [], + "source": [ + "import nemo.lightning as nl\n", + "from nemo.collections.common.tokenizers import AutoTokenizer\n", + "\n", + "# Instantiate data\n", + "data = run.Config(\n", + " llm.PreTrainingDataModule,\n", + " # Pass the path to your data here\n", + " paths=['preprocessed_data_text_document'],\n", + " seq_length=4096,\n", + " tokenizer=run.Config(\n", + " AutoTokenizer,\n", + " pretrained_model_name=\"/workspace/Llama-2-7b-hf\",\n", + " ),\n", + " micro_batch_size=1,\n", + " global_batch_size=8,\n", + " )\n", + "\n", + "\n", + "# Instantiate the recipe\n", + "recipe = configure_recipe(nodes=1, gpus_per_node=2)\n", + "\n", + "# Modify resume connector\n", + "resume = run.Config(\n", + " nl.AutoResume,\n", + " restore_config=run.Config(nl.RestoreConfig, path=\"/root/.cache/nemo/models/Llama-2-7b-hf\"),\n", + " )\n", + "recipe.resume = resume\n", + "recipe.data.tokenizer = run.Config(\n", + " AutoTokenizer,\n", + " pretrained_model_name=\"/workspace/Llama-2-7b-hf\"\n", + " )\n", + "\n", + "# (Optional) Modify the TP/PP/CP settings\n", + "recipe.trainer.strategy.tensor_model_parallel_size = 2\n", + "recipe.trainer.strategy.pipeline_model_parallel_size = 1\n", + "recipe.trainer.strategy.context_parallel_size = 1\n", + "recipe.data.global_batch_size = 8\n", + "recipe.log.log_dir= \"/workspace/logs_01_31\"\n", + "\n", + "# If not configured, the recipe uses mock data for pretraining\n", + "recipe.data = data\n", + "\n", + "# (Optional) Modify the data blends\n", + "# recipe.data.paths = [0.2, 'path/to/data1', 0.1, 'path/to/data2']\n", + "# recipe.data.paths = [1, 'preprocessed_data_text_document']" + ] + }, + { + "cell_type": "markdown", + "id": "303b9f780763d641", + "metadata": {}, + "source": [ + "After configure the training procedure properly, we can run the training by instantiate the `executor` and use `nemorun` to start the training:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1c1f8b3071d8ff80", + "metadata": {}, + "outputs": [], + "source": [ + "# Launch the pretraining job \n", + "executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)\n", + "run.run(recipe, executor=executor)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "77a82ff2-b15d-48bd-8cea-c3a2503190a8", + "metadata": {}, + "outputs": [], + "source": [ + "!nvidia-smi" + ] + }, + { + "cell_type": "markdown", + "id": "cf30d8c8", + "metadata": {}, + "source": [ + "### To monitor the training, launch Tensorboard from another terminal\n", + "\n", + "`tensorboard --logdir nemo_experiments --bind_all`" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tutorials/llm/llama/domain-adaptive-pretraining/code/extend_tokenizer_utils.py b/tutorials/llm/llama/domain-adaptive-pretraining/code/extend_tokenizer_utils.py new file mode 100755 index 000000000000..a1ef53920d65 --- /dev/null +++ b/tutorials/llm/llama/domain-adaptive-pretraining/code/extend_tokenizer_utils.py @@ -0,0 +1,275 @@ +import glob +import io +import json +import math +import os +import pprint +import random +import re +import sys +from collections import Counter + +import jsonlines +import numpy as np +import sentencepiece as spm +import sentencepiece.sentencepiece_model_pb2 as model +import torch +from datasets import Dataset, IterableDataset, load_dataset +from tokenization_helper import * +from tokenizers import ( + SentencePieceBPETokenizer, + Tokenizer, + decoders, + models, + normalizers, + pre_tokenizers, + processors, + trainers, +) +from transformers import PreTrainedTokenizerFast + + +def get_token_cnt_spm(data_root, tokenizer, batchsize, keys): + """ + Function to get number of tokens generated from a given dataset + + Args: + data_root (str): Path to folder containing data files in jsonl format. + tokenizer (AutoTokenizer): Tokenizer to create tokens from data + batchsize (int): batch size used for the text_iterator that generates of batches of text. + keys (list): Keys/metadata to extract from jsonl files + + Returns: + A new tokenizer of the same type as the original one, trained on data_root + """ + readers = [] + for f in glob.glob(data_root + "**/*.jsonl", recursive=True): + f = open(f, mode="r") + readers.append(jsonlines.Reader(f)) + + def gen(): + data = [] + cnt = 0 + for reader in readers: + for obj in reader: + for key in keys: + data.append(obj[key]) + cnt += 1 + if cnt >= batchsize: + yield data + cnt = 0 + data = [] + if len(data) > 0: + yield data + + ds = IterableDataset.from_generator(gen) + total_cnt = 0 + for d in ds: + ids = tokenizer.encode(d) # for spm model + total_cnt += sum([len(i) for i in ids]) + print("total token cnt", total_cnt) + + +def extend_tokenizer(vocab_size, split, model_type): + """ + Expand the general-purpose tokenizer with the newly identified tokens to get an extended Tokenizer + Args: + vocab_size (int): The target size of the vocabulary you want for domain specific tokenizer. + split (int): Number of splits used for original model weights (model parallelism) + model_type (str): Model type/family + + Returns: + Extended tokenizer is created and saved in the paths specified below + + """ + digit_flag = False + rm_subword_flag = False + unseen_flag = True + init_out_flag = True + newinit_flag = False + + tag = "code_gen" # Tag to identify custom_tokenization per use case + data_root = "./general_data" # path to general datasets collected from open-source domain + original_tokenizer_path = ( + f"./models/tokenizer/{model_type}/original_tokenizer/tokenizer.model" # path to original tokenizer + ) + domain_tok_vocab_path = f"./models/tokenizer/{model_type}/custom_tokenizer_init_{vocab_size}.json/vocab.json" # path to domain specific vocab file (created previously) + + # New model file paths that will be created + new_vocab_path = f"./models/tokenizer/{model_type}/new_tokenizer/" + tag + "_vocab.json" + new_model_path = f"./models/tokenizer/{model_type}/new_tokenizer/tokenizer_" + tag + ".model" + old_ebd_path = f"./models/weight/{model_type}/ori_{model_type}-hf_weight/" + new_ebd_path = f"./models/weight/{model_type}/new_{model_type}-hf_weight/" + + extend_tokenizer_llama( + data_root, + original_tokenizer_path, + domain_tok_vocab_path, + new_vocab_path, + new_model_path, + old_ebd_path, + new_ebd_path, + split, + ) + + print("Vocabulary path for extended tokenizer: ", new_vocab_path) + print("Tokenizer model path for extended tokenizer: ", new_model_path) + print("Modified embedding weights path for extended tokenizer: ", new_ebd_path) + + +def extend_tokenizer_high_freq_tokens( + data_root, + original_tokenizer_path, + new_tokens, + new_vocab_path, + new_model_path, + old_ebd_path=None, + new_ebd_path=None, + split=8, +): + """ + Expand the original llama tokenizer with the newly identified high frequency tokens to get a customized tokenizer + Args: + data_root (str): Path to general/domain specific data to identify tokens and extend tokenizer + original_tokenizer_path (str): Path to original tokenizer (llama 2 tokenizer downlaoded from hf) + new_tokens (List(str)): List of idenitfied high frequency tokens + new_vocab_path (str): Path to new vocabulary file + new_model_path (str): Path to new/customized tokenizer + old_ebd_path (str): Path to original llama2 embedding weights downlaoded from hf + new_ebd_path (str): Path to new embedding weights (modified due to tokenizer changes) + split (int): Number of splits used for original model weights (model parallelism) + + Returns: + New model files created and saved in the paths specified below + + """ + m = model.ModelProto() + m.ParseFromString(open(original_tokenizer_path, 'rb').read()) + ori_vocab_size = len(m.pieces) + + print("token_cnt with original tokenizer: ") + sp = spm.SentencePieceProcessor() + sp.load(original_tokenizer_path) + get_token_cnt_spm(data_root, sp, batchsize=1000, keys=["text"]) + + add_normal_cnt = len(new_tokens) + add_dummy_cnt = (len(new_tokens) // 1024 + 1) * 1024 - len(new_tokens) + total_add_cnt = add_normal_cnt + add_dummy_cnt + new_vocab_size = total_add_cnt + ori_vocab_size + total_cnt = new_vocab_size + 768 ## consider 768 padding vocab in llama/mixtral tokenizer + print("original vocab_size: ", ori_vocab_size) + print("added normal vocab: ", add_normal_cnt) + print("added dummy vocab: ", add_dummy_cnt) + print("new vocab_size: ", new_vocab_size) + print("padded vocab: ", 768) + print("total cnt (with padding vocab): ", total_cnt) + assert add_dummy_cnt >= 3, "there should be at least 3 extra tokens for finetuning" + + record = [] + N = len(m.pieces) + for i, sym in enumerate(new_tokens): + new_sym = m.SentencePiece() + new_sym.piece = sym + new_sym.score = 0.0 # default score for USER_DEFINED + new_sym.type = 4 # type value for USER_DEFINED + m.pieces.insert(N + i, new_sym) # position after default control symbols ("", "", "") + record.append([sym, N + i]) + + N = len(m.pieces) + for i in range(add_dummy_cnt): + new_sym = m.SentencePiece() + new_sym.piece = f"" + new_sym.score = 0.0 # default score for USER_DEFINED + new_sym.type = 4 # type value for USER_DEFINED + m.pieces.insert(N + i, new_sym) # position after default control symbols ("", "", "") + record.append([new_sym.piece, N + i]) + + with open(new_vocab_path, "w", encoding="utf8") as fp: + json.dump(record, fp) + + with open(new_model_path, 'wb') as f: + f.write(m.SerializeToString()) + + print("token_cnt with customized tokenizer: ") + sp = spm.SentencePieceProcessor() + sp.load(new_model_path) + get_token_cnt_spm(data_root, sp, batchsize=1000, keys=["text"]) + + old_ebd_paths = [] + for f in glob.glob(old_ebd_path + "/*.pt"): + old_ebd_paths.append(f) + + def myFunc(s): + return int(s.split("embedding_")[-1].split(".")[0]) + + old_ebd_paths.sort(key=myFunc) + word_embeddings = [] + output_layers = [] + for f in old_ebd_paths: + temp = torch.load(f) + word_embeddings.append(temp['word_embeddings']) + output_layers.append(temp['output_layer']) + word_embedding = torch.cat(word_embeddings, dim=1) + output_layer = torch.cat(output_layers, dim=0) + print("word_embedding shape: ", word_embedding.shape) + print("output_layer shape: ", output_layer.shape) + + N_ori_emb, N = word_embedding.shape + add_weight = torch.zeros(total_add_cnt, N) + word_embedding = torch.cat((word_embedding[:ori_vocab_size], add_weight, word_embedding[ori_vocab_size:]), 0) + + _, M = output_layer.shape + add_out = torch.zeros(total_add_cnt, M) + output_layer = torch.cat((output_layer[:ori_vocab_size], add_out, output_layer[ori_vocab_size:]), 0) + + sp = spm.SentencePieceProcessor() + sp.load(original_tokenizer_path) + + for r in record: + token = r[0] + idx = r[1] + ids = sp.encode_as_ids(token) + word_embedding[idx] = torch.mean(word_embedding[ids], dim=0) + output_layer[idx] = torch.mean(output_layer[ids], dim=0) + + word_embedding = word_embedding.bfloat16() + output_layer = output_layer.bfloat16() + + vocab_size, dimension = word_embedding.shape + split_dimension = dimension // (split) + split_vocab_size = vocab_size // split + prefix = new_ebd_path + "/embedding_" + for i in range(split): + start = i * split_dimension + end = (i + 1) * split_dimension + st = i * split_vocab_size + ed = (i + 1) * split_vocab_size + save_name = prefix + f"{i}" + ".pt" + temp = {} + temp['word_embeddings'] = word_embedding[:, start:end] # split word_embedding + temp['output_layer'] = output_layer[st:ed, :] # split output_layer + torch.save(temp, save_name) + + print("Completed saving new embeddings") + + +if __name__ == "__main__": + original_tokenizer_path = sys.argv[1] # original sentencepiece model + new_tokens = sys.argv[2] # new tokens to be added + new_model_path = sys.argv[3] # augmented sentencepiece model + old_ebd_path = sys.argv[4] # original embeddings + new_ebd_path = sys.argv[5] # augmented embeddings + new_vocab_path = sys.argv[6] # path to record added new tokens + split = int(sys.argv[7]) # num of partitions to split the augmented embeddings + data_root = sys.argv[8] + + extend_tokenizer_high_freq_tokens( + data_root, + original_tokenizer_path, + new_tokens, + new_vocab_path, + new_model_path, + old_ebd_path, + new_ebd_path, + split, + ) diff --git a/tutorials/llm/llama/domain-adaptive-pretraining/code/get_high_freq_tokens.py b/tutorials/llm/llama/domain-adaptive-pretraining/code/get_high_freq_tokens.py new file mode 100755 index 000000000000..70fcce20b9f9 --- /dev/null +++ b/tutorials/llm/llama/domain-adaptive-pretraining/code/get_high_freq_tokens.py @@ -0,0 +1,114 @@ +import json +import sys + +import matplotlib.pyplot as plt +import numpy as np +from bisect import bisect_left + + +# + +def binary_search(arr, low, high, bar=0.98): + total = arr.sum() + target = total * bar + print(arr, target) + if (high - low) >= 2: + mid = (high + low) // 2 + s = arr[0:mid].sum() + if s == target: + return mid + elif s < target: + if arr[0 : mid + 1].sum() >= target: + return mid + 1 + else: + return binary_search(arr, mid + 1, high, bar) + else: + if arr[0 : mid - 1].sum() <= target: + return mid + else: + return binary_search(arr, low, mid - 1, bar) + else: + return low + +def binary_search2(arr, low, high, bar=0.98): + arr_csum = np.cumsum(arr) + total = arr.sum() + target = int(total * bar) + print(arr, arr_csum, target) + i = bisect_left(arr_csum, target) + if i != len(arr_csum) and arr_csum[i] == target: + return arr[i] + else: + return low + + +# - + +def get_high_freq_tokens(token_usage_path, high_freq_tokens_path, p_th=0.98): + """ + Function to identify high frequency tokens from previous frequency analysis based on cutoff threshold. Selects the top-K tokens in a way that their cumulative frequency accounts for approximately 98%. + Args: + token_usage_path (str): Path to saved token usage frequency analysis results + high_freq_tokens_path (str): path to save selected high-frequency tokens (new tokens to be added) + p_th (float): Frequency Threshold + Returns: + Saves a file with high frequency tokens + """ + f = open(token_usage_path) + freq_dict = json.load(f) + + topics = [] + p_ths = [] + for key in freq_dict: + topics.append(key) + p_ths.append(p_th) + + tokens = {} + i = 0 + for topic in topics: + print(topic) + freq = freq_dict[topic] + freq_list = freq["new_freq"] + freqs = [] + ids = [] + for term in freq_list: + freqs.append(term[-1]) + ids.append(term[0]) + freqs_np = np.array(freqs) + th = binary_search(freqs_np, freqs_np.min(), freqs_np.max(), bar=p_ths[i]) + print(th) + i += 1 + if th > 0: + tokens[topic] = ids[0:th] + else: + raise ValueError("Threshold value is not greater than 0") + + L = [] + for key in tokens: + L = L + tokens[key] + L = set(L) + + token_category_dict = {} + for key in freq_dict: + temp = freq_dict[key]["new_freq"] + for tok in temp: + ids = tok[0] + name = tok[1] + cate = tok[2] + if ids in token_category_dict: + assert name == token_category_dict[ids][1] + else: + token_category_dict[ids] = [cate, name] + + add_tokens = [] + for i in L: + add_tokens.append(token_category_dict[i][1]) + + with open(high_freq_tokens_path, "w") as outfile: + json.dump(add_tokens, outfile) + + +if __name__ == "__main__": + token_usage_path = sys.argv[1] # token usage frequency + high_freq_tokens_path = sys.argv[2] + freq_threshold = float(sys.argv[3]) + get_high_freq_tokens(token_usage_path, high_freq_tokens_path, freq_threshold) diff --git a/tutorials/llm/llama/domain-adaptive-pretraining/code/imgs/embedding_table.png b/tutorials/llm/llama/domain-adaptive-pretraining/code/imgs/embedding_table.png new file mode 100644 index 0000000000000000000000000000000000000000..2ce90f866c73734ab32f648094bbcc7096b05027 GIT binary patch literal 69341 zcmeFYWmsfQ(kO^F?hQ2V?(XjH?%q(i)3`M5&_Ltv?(VLQySqDub9rapnQwOY&i%9d zbL+|ToUF{y%s5eTBBR0-lh!83|+L>Egn}L8xhW$#0QcPIJ7`g-&aA+T~-H}UD_$4B1gB-7mOA14i4GJRH zkVMc_=Z1!6*JhUy!d61YHuzIwV2DAs;h3T*E)E1B`~rfCEUj}~c6d-YIr`jAwH#)? zOt@LdgJc)iiRC(Lg8IQ5C2{s{reR=a3HC*S;3`5u7=qj_r^}*{(9(V*WXJiiy?g;> zMYq$Dn|}swew0x;di=%($z}{nX#SlJAEZG19!!F20qR#O;@tO3E0^j*TvIEOHMH6} z{E6k&HawvaD=KR;EZmT|904SA+&tn5Bz6KhBWYrr#AO&1dKJ0_jHs6j+SAmO_KMA*HQpYNE#<js=&q-S!?e!#|%m1U)&qVtR`2c*PmP0B{Vr;ZtoI@gd2 zF9;|cUGg6;nLq`|@>zkri`xXY7_MmL6O<}vnw zS`adF<8l1X%*Vo`+{g~+L3FqfJd$Zd&BJJ&g_th#1?kmsVeX zVa`8gOsuRQ%tsm{%72X&KZ`j3jiH$jLSL9Fi}dWO*Be&{nMiR*kBj$?W$1;3{7ZbczB<>FhiiOYyw+RT(RxU;@^*_nU z%9&NrGkl&Qb2#U>@?Ihi>M zRt!0@*+1&vIh;Ybu_uJ81gk_a^Ju1Bj?A4Yp4oqZ@CCmI=NQ&AC4JL?heHe5Hc>Z* zW!z5*Oi@XZN#P$yKH$TMP#Z`!&Zq%ha<`^-Wo!J_hPfKl;defOXguV=nf_%G^)MQ4 zpvFO~8EnmV4Q0(^jcyJ3f+RC&v2S(z?Fz=7h#$!yS|Cy&r0XZapaSw&=s^e#VM>1) zO)*L$T*7)3TjXG9G*Pi!s_BexVzOkmr0U2xVacH}p>|TF@t`9=zsjS=vd2|1F{ zj%K;L=5^^#D60I3=!iyGSz=Ya!Ex>kA-Wh_K644~3Hb@=3E4?3v-+~??}GGF-Fz?A z76}hoF{uF|GI27A0_pwoa8>jItK3m#ISo1`zRH!VXDvC2sGLES)!c<*SDEv`)}+>C zZiNoT4yg`>x4JKKeHtWkCT-#ogC=tIg+I-k52Po&Q4bcF;Kfw)j z^(6HgfXl^;1^mV7`RhfL?;?>GlGl)@$!W`q%g)Pb6}A*26vm8GnrE8Rj9X1yq?k+`A6OoEPS|O8 zV?<+6VW?qn&>G25$oTK;k8OCQc+lSl-C7;4HxTPEmDg5amwzc&s|c*PE!QbGnBYjw zV)T7^t$fC zN0=bwcW?#t3mGmBK88N}J8dt!v7D#|DVw9#-u5t1X+w!j2`O(Pmuwo3>4qu7X)K0FOte+HD_M5?8+xNrVoPp>FI1C^IlU{t?-ih)=!LiH~ zeS*QDzAG_G!nZVc&TBZHM7=S51!J;vDePTFt~O4FX8kqh3(BbsjZqwVR|R@G{Oz_| zw<`%1UX@W5eb|%$>7LU$zqyk+B+DGjkQwq>li4)pE37cAqMy>etVV~9>-$_|o?=P4 zU~$erPs1~42K$9}0)HzW#E&n~nwbioB&A4oCcv9|nrfK&>SER1h(;SnGZuON8YrbJ z3wC;Q^wo~ipU_&b5%--KWCI6a0RRg^r|}Cet~HE&z^U{fJ@KDCA;O0eI25vEA2QrC6DQ3WBKa_ z0&CymMQM~tR^=i=SbCaiS6^$D01RdU5jlF~uRz-RB+4`gzPgN36qQ)ih zMf_R3I)q031^Y^e4KR5$akP{Avb2zI#k2A3sXDn0xWHfK9W~EAFE*d$`}NhmYOt%Y zWg)-OOTg9l;oSnt7}5Dn^rPfG`=BQ`&;SVrN$QmNg!?q(^b`p|KyT@XLyC|mfkDvX zF?aYKkf&xMmtdtb&15g@c=j|ECIBQr;@)rw@X5W-zaHT$DlURe%Q;Y+FrlNOn^|~V z*qHS%;#pZ+k$*FO8_QvNwkVw}O{e3M;7#_mz45wux`xYzV_1DU55ybe#@CN*b1!0P zuRB!!Q$UmaMsILt?3~fH(;ulcE03&qA_HkaL;+qx@q4z{bP3D zT<0=i!>Nhau3__`#rp~Ibn?D4wsYhC=-tx|bMx26?gmavme0q1;29Je5(jTDFSS?s zRr71uL&D1DxGx}{TWSjN5#CuK%In!amgR8=IKJ5hzb}x-OGB`LQ%O*R!^LvK@#FR> z_fUjEl!06y<2C1TC{x3{O8))$-U6t(w)6*ZeLBK1V8^zB|Ksqg(uCLgb>t-yh(-V_ zVC7Z*BKk3R&-HM7)j!J6%z$n%(beb;_x$jk*~C$~AET~F0`%*Sy(2S-9U`b;7?j@+ zG?1O#-z`5@vVVVo%|g7RO^t8N9zb-*A%KXwgCd9bTZuA@>t29?7=>c&LmGN|{+^ob zp1pleq{0R5Zkt_Lpn9abXL&4|7{Bp7e#t@^E`6tahYNOvOq=i(_56%q)y*`e%;n@j zs6YGAAfO?ZAP}ED(9a9|^8x_@j}Hcc`n;olUZUAx|C)s)%?AHhAN1-kKp_HHuis^fbhC=e|BxmoQ(+G zZLDpbxZU}P{{g}M+5c;pftc_gAkF|jVofPhGeUNHMtVkKepo_6LS9Exb8cl( z@xReOzwr@&cXqbtW?*o0bE9`-p|^9iU|{0n;$mQAW?*Kf`-Gr#^00L_a;LL(BKaqg zf6);&b24$Xw0E|&vnBkCu92~wi!&cF@n4Mo{rRVyX6}~%!^zg^Z?!%ZWcX`_fr*}x z;or1BQF;Fw|G?DzA510|rvD4-e@y+~ zQPrHx97XJGK6yIx|IgO^&G$bi|3>6x_)GQwSc-qH^FKyE7n&cIm*L-=#t*xUTO0ub zA_yWSDx~TTdX@p@wXXIwxLKaIcGe-Wn(x`@V`;i3z7?sSEBdmd?ZEgFUXxD=RS+ z3{*nU|JshAz&76gK(+lJKnOua`~pbrApR>2LVq6;L801B#X6Gz4g(4s-wq||2l3z9 za})^Vh>yrb^nU~R9H*4L!T&!9dc*zhqD;`!f&kj&nMR*XI4m^%yo zuYyW`F2?15XYl_o#(%EO|2LN5XV-P#|CeE`g$t^f`3h|8QB5@b)*D5hm_f86u!ZJ=-_kF>iT)q(aYg^Mu56UH} z_yGO#ZV5u9rlsvI$KWGNfDH_TGwruUECq&tJ2*E zj~n$@(sfHVP7Kr|6R@Y%CaDXZDrtCmDb2sAWHZ1NQ)0DQMp-|)3mLffHU4wAB)5v8 z%hx+qA6R|K_6V74+_vuX@sWRadGO?rfgkr0wwM=)FM z1E>qc1XW(gN2@0T7@z3Gvg~4D7xLMrv*Iu|uFWruT>PK*fkeEO)iN7s-d7R}0`uA2 zvHf3L>2^*JE$eo?{-FyCs6j;X-lFaC>j9IGn9$8P29P!i|98?qno885Z_7k4pN{I= z9Yk&_HWVK?*grj+y#W|vmiF~SPFv$kC9;J0r+wi-w0n2tkCZiXq@Faef69X`%`OTC z?k8xH@Ywq^{^wt=>N!FXK%a9UQ z;ETm}4W+shpzZ7SrOHNHV!OKuyGdgGYvggs{!-)b=4XLs&%x)15ufSZ<>3044gGK1 zminhxZk>0u|LfWN-?y#hPmmuE&yN3Zsd({u{z%9Bhk)JN$X7QHC|!T-Ep0553Rkdi z-8mQU)$um7k*^=>Qhe~()FeR{fBY$Y3lJF?TU5|a#Winl-pan`+ZQ1KWIsMa8FK5- zXsY>0%5&SvxPP^8ly>s2sOQAygM>W%mvwxq(;?nHG^Qi`MpB(;{cB~A+OjvZSwJS1 zOp|uwz3_ut=cfm+<_A#{^7VbAq!XZSZmA4W>%-6nA%Cje$A9M%%`c-mS#^1=U?16+ z1>*5b=sd^p6&J=SckB{=M2&JRFUH(51q{3)SWlls1e-~-`_oex=8SU#JBLif1bOr* zY|Z5anx})m#}r;+u?c*ZYk>RHqvYGvBW5aci0@_Z-I-`I$EovjdWuF!7KmF{}n%f5di`j=64catIG zBa=`W?G}{t>x5eYztxnUTIYhRm}%~PwY0TNT;DG%m{Ff=7-jOt+rptl+nwUx`(}kB zW9oFhBeKA=2AczFxLoY61gAltSsUl(;?UEN!+@a4`YzNd**aOIm;5@0fICjA*5Z>5uCs%|Uj`Jj)r8OMr>mlD(xi58*o$H+ikwN1~l zv{Bg5n(K(o#Mpe~su-<*XpH#My2SuH6_LLya#*T4Ym!a3Yxub-aSoH9-rHxX>(jCy z80;Rz!NAi%Acj*|CgUCnbXUnGh0ng*Sj8s=|FEQ`L)UR_RJGHj6T9rBiY^T|;a}|D zBZ~I{zKO@udfl>(cwz`I63I_TA7}S0a+?;LzCQQ|B8rG@g)?*=sR_*xs5^%(ZW2b< zDwmr{b-~3eSxhB_s>uHe9$$}tSxX4{ikdgorr29J6+2q6DcO_8kk=WitwrtylnzUE zYFu)3G98<$vv52F$Z=D;2rMNmNped-r4OjKNWKYP^T8wy5tU82cDU_7 zC{99Z(5L>;^2O;te&-mL zgvBE&f^Sjpecf%NUj;R&8uwMZDRskDj+KSSqd?o4#1tSX^QwM}Ar3(%-6dR+?{8hX zW~uSI1Q35EMQZ*sv;z(sKNP_ILh27H+s#3%OIuT8{)Nu9t)3fq2M3@2eS`{?*|Ju2 zoL?<~S$uTT)7PxLHhN#x8w`xhevk4s*DBtxo;Bs9%~#1y*Q}%f{bHE<(Uq2bml|_so^~Hut`{7L9G-nj z8gxHDyrs^xmX)C}-wIPtOqhFb%pwLJPQk8kno{1`>V>vq;jvtS-ZQ!y6l4V6gk%I$ zm#mG!8P(C&Ikd|#um=DBDY^^wgN>Svr%zrhb*XLKiokDhC}&VkzH#{pw^w#{*oGI5 zfMG|Q0kf2fZE875qpaN|B7N#)^rpSkA}r9cqTLF=5ZyK~r+^+mjBdTvE#i@vLiNW4heYuf?hVVww{gH8T3{{IX`STz- z$a`DqNJX(_!UxoqL#I~dUD%GpeO1e04A*w5BRXo!Z|?}7HHAs~hfoQlQ>7CK#fH?L zuS3ke9%gKJcDJxUswtbLiXysa>i9WyumM|E7sR?&I6nS8g$NQhY7}J(e8;yi)x;yA z^O1EU>PSi@bQfV{q{8J?mOFUgNTWi5-)V*u%p+gyBU;VMiq+o-ETB3eb{C(Xjp60& z(-t4z@^=1QB3o8gGT&O-S~D&JdcOgLOCQBV18h*1FmM@r!fFviqm%uDNJ)iDl7nAy zr(0Sa9q?Uk3w7IVarLgI?qp$5IoiJKd{lnkBKN zZ`xc?fZ}P~3?C_+%{7=ZB0`m-lc{vmhx&wVmoHIAZ-4Y^Fb^VpT)swjSqkewgEvH% zZ#Fn}xy*ZxFb?tMIS}APz~<*LR?WUGOzXd(*^SdsdK5qUw}A^VAc#F_&h~W;>8{?w z!J`0H7_VXAzorca&<#xKW75bkK6`zJi~o#gwl*NZyhQfegMjMo;6+M;$?*}tU zXRxVG-q+Vo{Hhx;uc6DX+HGQR;1>@B;f%p|a2rPTsU8lzlWbDIk@pLTV&fvX`({Dr zUEGxWSpCL=)wurB98RDuJGaN|618tHwc+rm=H;cEdd^s^&9#|lm5i5Acqd?%(2Gsd zK@Smi&JZbLC-BWiR}qXqMglcOxgV95anX6Pnd8A0WdXYXyX;RmH41qK+7F%{KCX3;nDV~(i0O807tH`m%YE8AYAxQ0?tn7$=R(UhgSw? z-WJp@W9F`$VM1Jr@N&(JY%m60uT1$m4U@md6B{kKujLJx%bm-;T(n&bx@H3d92AD2 zkbZ%Xc2fnW=qxBkdWM1>`@7#SynH3~j38Yl@)wd7M2lS_a{T36BbhtNEkTQn@rIT# zKoGEyzbFt}I4WJDU$fR<4aaAjiUOy0w&XrvAh}nAc1=ka|KmP}>|(Sms=Fv4vazHF z{x-`~JY3+n-Cik7RR_%mD5!CWU&C=L4H56f3)y!$$+$aDABG+@a_NcG`5jF3pL^t% z=4!b6`|%&Wv&vV9JHhqIv60j9u8Vf0?sMOiwxu|}rKT7CIJX+{cqA$rvs!(HC@2`K z_Pj{$-X<}NrP~>O3|_qfjNRW)GCP zPOFmHkWp}}{XA=Az6n~6ze&&`B_taNZJhLxIrJs;#Q$hR0nr<>k(2$kN3cN{XEDy! z+u&1Z`{5I(R5{O`359@1PsOoLY{Y`xjt!_fH9e7ViJmHUAW^Y<|M?gkoC+vQqW^^l z+<@6`Q|}d5NK{hbrE}3?ml;UTN^4*;f~w}4Z79)h6m|W0|9rc|Y+-{@UV5b3dWM6+ zXKA$2H#C<>SN&+_yq>LwM5f}n?ZAO5i89iQZM8E}DIZ zCu;|gvEficu65B2{w(*GzJ3tn?-94y&t(yT4RUS29D3ipuRFGq*V+F*}}SCl;1>xGwX9{%&m*_(-WPbyfL? z00}2~2tr(1(NxI3X<_-&OL%&sD|pZ@dy{)bnqG_U#@ZbYbb<}k3AHp50s+lo)H56E zp6ncT;T#V&q4^gajVqkO4r*V0ujt!6e1TF)ua#;XpbiImfLo%s?L}MfO1tf9*KR)p z&sujw06F(F^$Uzv(#r+tz(FO#Pmv=&i$@z~{4l$@iFK_xV<)8rl3LzNZ_@r8#^b%2 z)flMGJkK_YMhP&|t!s#Ox}<1>wJ69Zams52*#O4hOBiD)6u#8#Rwb|IHJpY6MGKC* zneIZw@|hh5;{r0e?eM-=L?#}|z#A6i#74=xcBd%6Zyqzp%6HQ|c(Q#8oKnK%vZyk` z0bgx@pb&2tAd4l8A9B;9>!g!84oZoe!eg`j=+caQml~;tmrWcOSf@}sRS(n7DcR3z z+(*N{uJgjKRio32;SGg9-p5Nmy7xgrA%5OhR<`TV z3m8&JtY}&%oIpU(1dr&?aH`1{xT1irQm>1qpC>yW;=ra^X2YC3My?MCMo2bt+aN(H zn(FD8_DHW>m0FGqyz@Y>f|^SxDdTjk&RXAW{hyHd7ZUF9QEF2JMQ&2?oF;t^^YPhm~X?EVW{ zRxsr*9O&Se2(J5De+{_!5cr4`G7SZ6`j2&cp8k%iz4{yh4x5;8ei8mDuMue14dlb% zAO)7+SwjOEBsdYBC_Jus%SgshdzhP;V$ z8`*eINLdh}Xi0Ez)x?WuQc{P=e5SlCBZa*U#Q4zh5=1D*tV#7t8b9QuPgVwxaTCHT z9?To;t-+9d=bDJM#tTr<66vdKU)fF$yK|>?3MsP4FeBg5a`k)L3R>_n!q|~-nXb`> z1~O?zh!Xa$(KUMG6~&}Tp%A2#;KbLdNb(doOw^igF*I{Zkqf1?=TK*~?mVQIzC*XT z4abIJnbW3*7Gs&z8uDsyHXOaO5=9T%%z3zJfYJL+;ft81*fy@ zxRMyM=VS!M!Z&LO(B^sy!_!W*z&J-vt>j^w-W)sA@n{y-1_cYje#GmSsr7E%Xcg6cpftaq!1Kx^d4|N6>K_3`Yq{4%$T>ixh) zPHsBKb({aV3yj)N(S{9{%cAnB)(2FU#N{GwCXL#4C2cvNk-o(sUEn}vKK00PyoZ;O zA{*yt>L+#HHtxNiUW%*AbUqyp_Oh7XsD^PtEgv`996O(Hw}-a9dP5f#X#hR05bDQ-mrAfBrO)b(}2UQpZ5o@N)D{ssRRvSh371FK@!qyEt@^w zjALs!Rz_(bXISZyk^#h2FBHKv1{)y)R#CU#7ai8B7b_FVyS}!sk z6w7NIY-tNFw(TKRxDCIQub+fTQcBe<_c;uVc`Z@Fh4>)g11V&YC(89Y%pqyjR@W{w z^wfGr*$yv#nR>glkHs@Q@q>VbQmyaeB^&MPrO3+Lf?^dqXx&WK3sU0F6}mT3d+)cG zHW&p%SP5))eb#cS` zVD3lH=(V>dbK!7(|E09?kvkR!mkcqO{wb+sDWO`>x5x-r0kEcZK4h(!gc4+u-9QOp z!gS)3}ip8{M>v>suQ#gI)d%gc=sn zg}tu8_Xzu`$waBunU%y%&~Tqy7dMGI^hCwb`QnC)a1yp4De3OUN%1c!T?|O z{V&VfH|IW!oSW6cqyqHEsB3j9QIDMsdrtkEUOmkbMcveQdqe^l==Ks~SwSv((ODr? z8Ob*_m@uEmjbbU*;SgWYK+(}j%rl5V!>TD|@r-FP+434-6Qc$OkdXz@s=B=?+~&*F zr5^muFN+Z^&B z6w)dEMP|u3O%mtMmS~A~$$yV;P>=p<2hxicIa)47G>6>3=!E_>8Mitva+#m_G34#m zi_k(h81^;=#9rL4-c^YGGcwep=ayc-Z~}7{m*`#8fNtPxbfDDW(y`Zd59_0G!E?`R z8c|$EKl-AD8er!pmat~}b{?brFi^BaDyX{FpjYTq)2|7$V6#rmbIy)*?R=^MV%=4Yon;50I4(Tc1bv2P5zX$~E ze~)Cvzz`Jmsqe$e)wr=6dra{XiP^PNH`WrH?hK|@>4ZII zwVCZ1e3%I%eTSDZ&^&ZSCv94DM=;K890)260I^(6qZ#4YfOlX~%2nV72fJ6d^k)OO zkvY#epn{eeqj#2dK8d@&ZdMo;7XgV~xY`OzN`;vXGjJK5Qy`-8MTBa3LcL=CKFnTA z@zVzPsozzo{^13G4TZKH;03o90&Kx)LwsxQ_dv!>h;&&A&PCcg8GTJ2+?%f3!`^(d$Lvft2RZExm+`>(~ zs7co5?o4ifZ9tyu+pBCl989_Z9$+WiTW*vgtDx9^@!ZaBwZ=kmp%&$!DL)P0f2;6~ z<(A+RU1Y0vhiA4uPK8+sPf6j{w@|uv3=B{HX^`4*%8GkLn z;1`+-gT(mYcg^`D?0E&`Y1zxYgkTZ7vCO*2`PUI-m+DZXBTiU}EIsVV=!JQ=Nhc{3sqOw@WI-mQzB{qwl3q(6LWm?s>6UcB=+} z>c?)ENPVoM;3dfRK{Zx+ZH`651pDguEhwY$M{{Qff)i*a=9I5St`QS9Xp@tk+e9fouS@BZW}UxItTp9z zF3mXLun9wsG7vB?w}R;nA$SJo=aN&Yw4AGuZ=NvL;rAnTZ&<6{JI;@S36g%M+P$JO zDIRGvA=o;~Wm#YLt#KmcwxZ8t&OIL6S9O<;J7VycH5U~95i%(oeSSwH*h0DuFdrPF zow_nDfS;|<1n$Ct06L!f;6dH8H}%mtkAMfL8@h!CZ{ph=0l-zI!^RNY4REEaypCqo ziqS@qNO!LdB9t#fXHw{hJ3A5#$ZvhpK}D5I;GQ~~HEaoAikr+mEnhsDDk4B6NOz#l z$hwXDSJg`Q8_b?CrEc?!5tr|t9g%%OAMB;JCP!nKc4eMIP;7B!b1Mb>bwq&7LqjD2 z4uGox_cM_0%!k)t`1;Kz4@Fj!xfxF&ccea{SfE zzsPf&@T#6TO!lj>L~u{8Dq7B|5$cd2LVU@&~+_u)d?@!|;(;i3Gq^a6}=*6FycZZlkA zeY;KZ#5!w0SSmi3enCygbP^YqZmOjYm6HPa`V7ow=` z@r#3=DNqdcLA>1C==vk8~WIG4Y{z~h5a zGjmJJh|!&>eg_1z{h=J*!#>K zD{ z1)I**O3)jiLQgvzuwkNDgEtr8z`>3EHuirDZ*gRvKv{;HcWAHewbE+BnAh!Fpg zbr|{Uw|O&vdM=EsG|}3*_x5IpO2-Q~eHzswtQ zGDRs~&LAF(M;l#>Cy`Fw5Dg!ay%@@dm0_g|HoIk{}j#kJdHQBjr-_mdBnA zBZ1{4%{*J*y4P+qVzQWJnX|+qElBMq7PpsZ;Ewh3!#Y2iVkRQ>e4BsG~(dsrSa$6e=`d8m9hcG3oE)lT86I_4xAIDqeZOSelKik{-` zSbh50(XLthe&boP+5s*+OAC|X&>qwc>0Ydz&-WIz*Y!-SJA&j3KG$-u|HB0&=f zI7z?GBM=27Pd*D!?t;`tP%ad5RHdFxe|s_)ys=U(=^CDHmPOBau~AtxKKV>7tU>!W z7^Gm|IVAC~yp_MdNH*5XVHL0)KF%J(m^{<4ma*-^Qmx*pQyt~1@7z~dIn7%it&&9z zj;DcsF1H3!EmpOx%Th|?Tk_>htKP8mEBDSt262v*!{e)NXo?Mnt4kEReH34}{$>N3 zoABFKLGTdh&0|)BB*=@H&hA>Gy_;Mt9xila2MskXH z6{oxR;(PR+$LglY%gUmsq35wNNv##{;I2w#{9`N=@};ohF*iDK7QtwFu z{FKUgCO|5BrQRAV$m{Mw#S$Naby9JaANGsluL1#ahNNP-TiO-B3jyj??DX5~u(|Ys z${gk{Eb2lN1eJp>)rtU{56C6F0At|ZnPnIf=RZx^$3?X!Kd_?Npp z<<2X#Fs={bP;Cw3S@!@@VZ+j(W|a;z??sul=LU|0Vy$F0*WCBq%FQ;3vufRX;?8XT z1EKE?==%irh}f9ZN{DWbU%NuD8uJBOL5~iSI~SBtWYO>~nJRCa2d%#$DsHNt71*6v z>A#<2qBd@cB!p6)jMwqhDwZi~H(BQWwlh)lsJi>+ae_N-y^K9u|5xe8BpQ;8i5k{g zK8b_xAZ{GtF%H%|O0?kOz`cdg$DcpP%WzH|WWyTSz?0dcONG}dS zNz(up8VxKBQ>e8tiGg(3(~+Hi5A0U=dr6)1~`j6Jr>7UO=D+9-2b?CSz!DxA1MN$AZKso-yuZhco4{#3cWL zv2lRO_})(UzC(aTlZw)uw}qKlL8vf_NymCoexhE{tNx@Bfn0RmjRSgkpZZz=V_cea z#AT7TcmviC!6NxzZ3MpZNtaw{0%pST3tsvLJc> z>{1wxZmd0rWf9h7*S0k8lhf(~pp~Sv+AjtaE`+ASQ+YjKO9sy!^sejcwO21p=8ovh zx#*v!1&P~RHVURY(fzQyld{yv?$~Cp!~vj-yG+i@bpGlX2I+cH01~ymKy@rMK-OTh z`4l9@13yEne10GdZ1C2`!Ajog_I}+*6~IoDi#wtgF@sZ#wI(s?8E-XezB>FI`$kYv zgC+Y#J8E%_waNJy#17Pba|_{7Ef^&-1p&5MzQ!@cjp4K&b@Au0jUGwt9L9DIEYM ztw#Gn)z%*!FaES=MLq&nBrO8tE+Tt@DI1@E8mn}y9)N|x1SzFBm^P4n<^0eXpYD>^9AJID)6R zmzyW79*$P z8Lgqb$E;G18}ox^I&@R2jA}B%bXGMBU^VCV;o5bU7*1KPO%L*mtbGICecUA*)vdFu z9oNEoWqDKl8mY)SN+tncqQ8+b-A-~a@%-}H*p2CZ+aUUB*FlX^lu7SnZ}xdU#%*G$ z(O`^ap8C}pp3=_ntXN*hJ?!o$d+mLAmm(K77WNsc84Uz-pz!97wB z0NuTaKZ0+$#b?y@^Bg57QYn-khr|@>j`~N*V5M-kTeDvpBUJTW1EIMo^2L08);hh| zWk*=P@M*!1d+7u*p1bDbc%Myzg2#zPqLaf^Q(=@k1s11C-)pRrmFmKw+ufgkm$7^n z#tU_C%=w*7XZ74z=gZ1BH_v7#oqKDLQ|L#MvK|kodWXLn-t5-0f6z5!FA}CbkYa_M zO||?2>?PvWYpf2U{jINfPbR`s{#+%TELCLv<*bu!e;?B0_we?sufEf!utiZ&I-q6Nv*#G+=*Z1;z%>0NYU6f$Jl z`A_#tRcJdCPn*d~gc3?V?gRIJ)%3_9cD3r1b!a!UG|}!AwzX924FdyC0w+D6xm;7e z9ZT+A7IGm;UjBWr{;IX_7cFQrl`LVds%r>hs5yjPgliHRefr+MJ0!K@mn!`|vJyL_ z6A*S{V5vK;c+g1I*xO*S<8^$de%ca~vMtyu_rjX~O=Lw@Xcr@9nv3wEamSAP`Mx7J zYsvQK@?nDN&)3(7X`D$}=RT^9Nhh0}wBAB~0j36uW@(>086qZF(d3x=?(Mqi`HBq%3ne}b9vAW5;y+B~T>9j&{KuV2yw_O8;6CSZD-PL3 z{MfkSJnRf6Se3b>x{&f`A(;fMzOPt0R-fGG@nIdZ6ZAEk&Ut7Fc=YqUQnuHE?W%GA zv0purxM@Vk6%VSzjkSqQ3Gip*CWV-qM(Cd(o0gFSTh}&4<>bE+Y=c7wXGtG=BgkGD zkgxbNeCI?cKfH1*6%NrnkIhhhMoR{4G+hjaPbQ~EcKud^8 zpwp_os@HUG%_qw~f>}@oFqiA}B2}@Re!lMJ6-qm1dTGMe;HTD0)HzC|M@C%?Gl?@U zwmgf8;lQcM35p8UqPced{nsUbP?_xVPDS;-4y4nLX=INb?gJ&0s(81?HL68n6v+ zYOv;);L*Yzr0tcprRmLWUZpb5C{L`2s*7+2+Kp%7*tw@*kyvhc%a) zW#vc_1I+|=*2K}*3gB?T?mNsh8^ks96jMOT0!xl#zfQD5hdBS3mryEX$T{+6=khIV zeCa4fQi`sDL9*&7wi!>U8rUSrr4iFV|Ie(q(hq z)XG1lCQ2)Jit4_{P<{J=2zALi^AhBr`mz3`6oX%8ziCFy;tDHjS7y*Wj^m)v#G80t zS<6;id9^G5R$N`i>+(_wEM`{XI3wd?5L#NZT3JzQT{ERd5`Af{OkjMFHRYNrU4sR^ zQ40(C!^``FJsuGu&Pn`ru)x=)#E#vV7ji^?$@3l(azuY%WKgpE7KW30bb^#^AyYdV zue|to*C4p+q7M3WD@~LcC*A9E4C!$=fb9JQVCaBp(y7{AhlwFQkUN!cx-DUUet|pN zr=P-LmgNfz*BOWWq>d#sRC&(9F^5H@sHdK0wK7<0Lj88al5nn-5EeA~zu0>Vud3Fr zeOM3`M7p~hq`PHPQWDZ7Al=>Fu_@_Ny1To(yAhD?ZupiSkIy;Jc;DYY@V#T~!C1op6$20^jt1L)i^6Hc|v{#0^QrjJ+{do2uv24v&b_Ecn?bV})Du{!EHj z+iY>Lw%|Tv|1JMQT2hq+Jt$7J2JABU7bE;(96lDh_hx3SLx92e*;pkma;T!ko|D=@nLKqzAX~Cf4{Uhm$WO% z)5;}USEg=Tqy|sZfvAigJ7rMU3^y=hd@f50!M7^664*q+QKdanaKB$#Ex$MLZIyWl z^EhZyn}_DInYQ4RUnGTVBK9zC2E&bHqk7=8^lecTl$R_@^Y+~Z_%*Z+&UfM`Z$mj8 zIS!l#kP1hbT_Y)k8NR(T;gy1 z1~>{us@$1IT}L4{Lk6wRj8BkLAO~p4s=!D6+>LQ zM!0(|XWSiPi=s;^(jpic4RF})>@24Dr#>N`uSsPeJPRCh=#QJTNVKZ2cKEslXL(ZM zZ&yUlnJ>s@YM|1%MHU!D%p6tXDXbT$5`sOcCV!lbI<+5LA-ZTpOSWKt!Jyh&by2!V zo3E^o%p&|U8;htsFH82dagD&ao2>m8;vM6=Cat5=0eKqw7+BFKe8?+Gne%+ES+_vt$wllfWT2s;e29CtPTcTVav(8GD8c6U@LE^Z9z|73snSCh?@#P! z3}GBCZ838=IzPSzR~M|?7K_qMXQ1gM?m1&##^+!qxle1d=;sV;c-+bZxQ&zC7^fvvQO=118du=w-m0w14b`jHVf%Y!yXFF%jz|VOn=-fkCHu6fQ zcpD9f;C~LsyeAc%rEM6bnh>*v%&-7?ntvpnTGs0T8bMsIMJPM%T`XUkw@i46mfb#nj zv`>eOuiQ039>eOJTe#lEHQnKp^@%XeFgi^Y+XlCv!-PHU0#o7U`I$0s{Oj8e%KeA; zz&}pD46c#n4cGrh*`pRv=C1aQzN9rUTo2PbjE7?Dc&cvC35RFvhjNQfuG_zp7eFe7 zHDwDe?r@n_iNOkRE4By(8gGcu+7#)wHIq{_;Ct(*?k$+#@sLxZdndd$s6xEE|CE3U z={0Vy?uS60MES~NrTP|Is~kRk!wY$ugQ8^7n@$iB8i!Du4Y%C=E5W5_uGXuXjh6xX z5r+-w?Z(OixYH}(It78RCU-Zb*UCTYsbo4-ZSrpxR=wzRganoxM%`IRUJER&PE!sd zcxOqp<9ak6g~w-K#8(7Qxd&R{V`0MXE$T3syo?rr4kA*ncwf3mQR;vfQY_1`YfRhs zUd=!kuBgDHLl~;nWMIMDrS46-<-&@S#sb!MiHWt_M&BM=?GOvlCO9N#Nk}?+1?4Oh zv41C9{tB7HAF31!2Al6a)8UOzUvANK*=0HLzFZ?2C98%A-+ir>tdhj==ZPMPil&3f zJ4~>nfUT6EwE!*3t6v1YHHRN5)WR++L{=z62 zR65-_re{*(*Y(3EWb0k&lEJy9GG6I8BiVOzZ&FFUUtOoaI)cA=k|PDD*cAJwF-Wu< zq)?u$w7lPc@xoFRf%WaUF=w9n%nN!R3C5_JH-ERY0o;(MDAMR`N(XUIk{;i#e=#HJ z9?mMSU*QBq^J*a7j+y!b|A=$4PNrzleObbr7k#=NDNRwodMs;+AnbOEmzss)O&-7g zJqsXTkq+pz(2Jg=AVez@Sbez^I=GGR zkV-H|##dQeLsY92$L;EpYchUOE?vUJ$OQ9;i}`gtJVBn_{eyY)xS~V4dqvxQ33l0I=d9I+hV;f&Qt<=eWBis=QY5Nu- zpoMZ=_VPVRl~;53;3ZJrtqwgI>|*As?%$pBTcZh=r5<}Vdf=EWUI4V|2>I8i5vMAs z+9^`1-(9bxHBgW!$XVYk8npL*Z6Zj(N&7?2jO*WBq=z8FS#G4hy+tDsF6_@IKj-t( z4-m_>kUL^`MWq9pKwmpHet~+4?UlL1oYlXknM3?MjHpKfU~BZC?hR6H^)O1=exKMm zi5C{uS?8DlY@a)5xEMGXFa(5XQ?KN`$Ck98cP4ky%bs_TZj#!&WO2bHX3$I3ck#?g zK$n>z$p`;{3jAwktH0ZO7mzuQItc>5^t`88s$Q}W<6uB0^vWO^9bcqg{%B==7^Fm? zA=Ko`3ZUkf{hMN~DKOV0{y##w-AQgBywYLJ|LWE*fu21{3tjm*BW@^Zs%6(^|GP&1 zdoIki+Wkg;^kGaQ5C13p>WuEbm%w%xLp>i`6r&9y3r?Pr(@N6R!oc2{YILt&2|>Ea zXfGx+f5rs$@w7ZseSgVoeklOy1NOwXSI}9Px9;Czx{F=t>X~sEW&G~@*D^2By|C7$ zqmM!x)OO2#x@2Xp4!o+Iap{1rb}wj_wWo^ueZ)GwwIwFA{r%@3x%H1+s?u>Qmgg9O z8yHDSjaGZg@0%OQYPCzCNTO;)V$V-bb@&!_tBLpHpfcXWf#2(m_u{4kC;#FpwMUVZo9xP5YwuO4N-dB}zN^AOUm_ED?1wkK#Ct6bbf{}T zL;he_GPg`!jm?t>7X!7&!D+F1!Q}Mg#GDi7b(DD>EiNj|If9o+gH&wuapV4t&Nr6L zEkUVVj6|s`1eeQqkg4wkWLVW!A6N0v$dJ}4`oGW=e^KCZb~^Jwkx|hd_W8MymndHK z{@8P+Hg)?{i{w)mpVthlnVWxT*F$h9A!D{eH8wVNj`V1~d!9i`=bkBv?c}Ze;In8i z0yN^=8~uFqfA=B&*TpdpB#YK@BkM-!f9v2M#ROXLKSbCrWUaRT@3))*`g;C`J*TQf3>^U&dXmo{)ahJGQ=Mg6i6i$-y(~K zbVO%&)%^rIt)AqFIB4`01}SqHoB(D-V$%dc29QT2G-gC+shY0Sn`0f7!lB3*E`T6;z#BV>G?kP#g+vpy#>~41#62U|%m4Dnl_-M}6zl@`(GYK8k*3`&nB74NT z+f65h8|+fYB>$SmzTt$@#lX;=?@`C(^}dxZP^lRT#)xVqf_8KYYRZ#VJWQr{31-Jn zKW%+d_861w7FpYqDiN0wnrlrPCDM<;FWjQZubc7beA4 z#=Y;O(`*M9Gd`ccw|8Swa8;hsM^xSI-^Qp#Pmp3A4jVmxj)_PAYh={_45T>Zfa2Tm zFPMALzaqci2>jpI=kdV!GyIHyxad6}Qs%?lp|s5aEK zZH)LkkN(GC{JHop1|T*bIi12keJX!1@Mk5`QQe>Q!at?(-`o7>-f^jbnQo%*Loa?~ z{yz)+rF#R?D7%&U)w1Dxn-Y7&%>m>-QB@S6Jz7d!a_-G9Goj5*^eKmPBcYC z{4;ZM9R6P3t4$Pm@!!?X1bNr@_e~S{J(~At%Y^a^3cB`S|#B z%9KGX4s^1zvUv2OVBEQrWSM_CP3=;|4;JYcdovR9^3f->;~j2v&aEXbrb{vn=)Btw zytCn8aBKj+b8pq_w7>{e( zK0ahDER&7!Ffcp5p%ir~md7vt@itoti6Ww-NkTB_(cLELt9%u{$s!zojGs(T6;*9) zYT|an*)icgc8DFz5amug{fjivjsWUrImIyC8KhlS+xgOQPu zUy>jXC;Usw;X_(=!+FV7R8+dhWf3lm6bt4=MMXs3gFwDULQO&E0$K=n7d8g6d%cXh z-5grK7xIMi1*RK%3krSB&F!rXzrMR0XmmRBeLLf$zr!i}kN5ADL3rFpk%q}Ahj1w` zCr8T4`caziMvsIm|GM{Y&f$MmQA`k2wb_Ece!-_O_lp88ESg?H7B}gbU;F=x0PT1s zm}DCDuc!IfYx9Jv2ehPIjn*~#f4}J~{ZC#5HsAFBb&neGHXFLy+5aC(d+Xi$Rm`}; z;Yd@ZLO)E;z(B&rrUE1^+!u<)tk)TcbjMf(=*+=F%|I-jT4rOTdV702DIJ}ZH1CDN z$;AP_l95qfp;n6~%WOq;T59UsXmY8j%*@P(`5AVWg^G&i@bK_A1bph`(#e!oR#t%5 zGB#$iv*YRN>WYewr_}8M#{lmK*OKRswVnN6rvMB!1B5=SwDbctHTB?>prD{WKQ}G+<)%*YJa#$g}{3rL91HX8|Y!*pmBeH zPs+oiF3o+wa&uteVba+RhCK%PwF5&gWPlUYXXBo4o+jXTZ_M*p3!F2## z)W!dJe13Y$l$35Xk=?iCutF!FBYlv&GCOP7{dl`qk`(sUd>``p^DqJDhl`m3)){qO zO$YVXf0-H5Q3)tpi~CvKa6oijb$of3w(@gx?iL6hj{}$thLgdm3Z$+#8*!f>?{79E znv0926&fdf+=^UIx-cYcZEZ!}+|os!%kZ4|(khnzea87R;t!3sHAU$0c7Oq;UXayf5ux?eGZt?M&CuR76!O=kFj zBF*6yaEDiU?fYApZ__+`V515Yi=syOpB9tT^AYosu+T0i-&xHRYXG`V%)V&pvz@oU z@_5Pa^mwz?wQ3odnz)-S!Y*I$H3k=JUb*aY#A`EOAd!V0S+ne1V5CJyQq`Q=xl2_( zJ%zG{q2(&u9fo6|Xuni%r~f>2E!TYed{S9hbSZRa$L!mz)v%IqW!`uqGQrmJs~Yfc!zN&X?0B{0u(9&=STh%rpA&KDaN()t z*G%7XKfWI>!~bAQ0>8_rdR0_EGX31)0&<+U@-XN^zF{9zQw(= z6aTm#&i}UM@uFl%b6G`eK_Q&#K*&$=!czy!5h^NUqH9^1|M;c5F9{e%A_CE*HnVI0 zXv)D?3NCPWRqzBEAC;SO6Ney@{8PM%t6m~Y4GUc01!tmQb~hM}z+wdeS^ZSA*yN0# zf&l#E1@Q@44`%P=VD-fN2USio@!94OGfr}}G*_Q#`Z=n7Ulf_X&*@Cd;}mpMeVZJS zEWw4|<+ubr>mgbFnA49)(NM9QDJ8AG@hk~$HhTKTWAAMj=OL>eJcD9cuM^-qtg+F~ z&IR6?m*0dJWW5x+#CW@!zI(%l0)oJ`gqs(WWqyCycqV;z5Bw<)W*h-DDT{^!wspQ{ z%hh)_?~5RJX-XQ232dx>FUpX%vPHS5$`1`&T>AMD~ z)s&*~CI6Yd^&nV+U@7@ZNdhb9l9~fBoWC-OUVB7B6mH5UxMOdMQ8*x5Om0#PE0qK5 z3~#Q$tU*_|V~ygw0lOR+#~quk3*G@&rzEV;7^b^kIy!x@Thk;O4rmraiRW=DmyBm< z+s~K^5xq)tE-k*(U1Hx`t$Q8ioLj6ZzN?Y>EfUQC=tx3pjG0-|@;JYrVJB0#%xgTB zE9^JyuARkw3|$;DL2w#U<5+F&B<0Pf=v;P?-lt~Af;u%9K{O{;WYs9tUC z7JZ3}Gq{|a_jjf?n=NgdRO)~%m@?kobH7cpXl8tugN?@Xa6XZa|9+{?-{mwB{=Clb zRaf8S`P`Zz{u`G)SwvPjy?Meplb0`A2P}K@qA;r*zOG~k;Ckp;dUuLuhSbE>009rD z-h-E>;FI}yNt-o&&%01l^8`vkh-t58+W_2_t{G}zTzEjC!`IjN^kT+^!n|cuLm|u} z>!0H^$TLL4*>DJ#T5|^i%u~-%?fSn^iKr`JpyqUCy!Kmhzbh)!pOlJ9%|4u~5aoNk z);&w@E|#r1W*+9~NS>BX%Sm%LF2p)JI}4+!4UX&U>pMN}K)@LgubNeW zI5*(GOF$y1dX(|Fm9G$=urj(iOM9Z_zv+`1w84u5t8_^(N$xnR%lJr?@{&)dp5bdgtJZiG+0YtF8hp;w?zoi)i~^H(A;d8tx`LW4tmy+5)rFK z5sShlx;TmLqxMnHQ!kvx!o{=h?g(6zdb~HamZ8T8#~`(D6DhlwDMw5m_Z#i>aR z-8JDQoBF|YtKL$%eL>A(JodygIiwm=pO;ISC%nF$8xP;#zd)-}T zAM89=yM?1tcE)^#qfQ@A32#nT_ARg6apIMnqT1|0QG=F^5u+}{imgiRAslcEL8xJS z1oP*<=@)X!2}`ev8kC}~R`ZadXM9T7o5*g^>JC|J`7noIGK@*7Nj3Sjp4e!mc^0f0 zTImI{-*BrGGUX$Pm5U<-WO==eU6@5N^W<#Eu4Yld?rk0Z_q~~oIIXu^$u{6Q8t$us zdiH6(@cNpm^iPqMxeVKjqI+J=!QQG$EtPO+f*lQss_F(a1i2$Rad4eiLsIV<%kR zpq(+fVDfS>-i2$l_dDv?^W&5o9OwB?&w6Q%IET%W&Y5lq*2B~{)mc@-=|bzIHF^Ph z=w>_OD9?AXJqYNj@`kZkuB<<5)S-ScKnT8hNmAJPWj1vM45+q!fXI^2WELo-KD{HP_o*j0aNQk+f{z$4>YC%D8xBX=LyKIweK{FxdPX&Z|Zx?sR%U!O_DvsK?5=rvdE|3obEp(71fZF5tO;!tljxrh{2!q*EL$5|VScmihe4&M zT-evpj5Zgxg5_h#hKu(+D_+?MkI_Iu#;CMF7vGYPvFx7cQj+nQTRDEF10D6Pkt~ut zO246Oq%0j_)%LO~$_7VB^|u3He5GQ{uKU4YE3fG+Lv+=Giqh4DvvP@x6JGSRJko>_ zUYOG~=qdM1U^tIAGmoR@25G-~d>nFI?S9&H?v{&gl2^Se02RzY(vDh$6bCfS3tlNuhVWI_T&bfi{^TBn`;1#zT3Xs1&R#7`YF4+|Om$W6>|E70 zW^MT4d%vH9y<=n7wMsb|0_VM)Z0mk%@#T^D;s2gE9?=BbGIX&Ug$J6Df?oY_cGb z!YQPSDHukeVza=KO+L9Q9dSKv;cqRw9D2Hp+!T#$aZI>hR$N_i9;$5B%&4kCfdrt~ zc)gww_7={!mVX$K@+P4v;o>F7dOd&TZ?)dt)zz}?i3n9vze|n6;FMwugw+0>NtUkm z{wP0;D5299txaU+p+(dj(#!YK(3TeG^&AQ{F(8l-|9rd}k+tPTI{Dx5pMUj%e?4hB zIQ$c@N(^nf5u>^Jh@{@NbGuulkJZ3NT#xYB^J5b+%0^_;Uxn25TpC9cr#9m6lsHJKuZC>L^$`EZm43pQo6gJtjSx3`q7OVj;w?aOy=9bk5qxg}m z>!i1YGlPQJ{7FFbRV9z(%P9;NGT5(?;xHdXCMvNqxK)-@d!=qy?@{++gFK-F|9o1z zNn1r2CS?qwF+qaFNLOxpJz4@ zrO$E2?P^Z<$zZPh^QLK?7>DilD4bk_l{Qdn8R0!EJBFBJcyqcjji;lZ!ewVlVUbKS zPZk^+>Q!kTii-Bye!cP2{y5%2(hMRg=PNU@o&SCma?c}|iz%gr;pyZM)E$7xFyN%a z8NhW|w7SH%z_p+KR9#_LiF7f#M1RR7_Q(gB1sen%1yZWll3WS$&b;dig=^tZjgAuV z1!X*Pd6T+{uuq7a=4}0bB0@-RiRR;|@x(PsaxwtzI5kNEmP!DgXUC%F z2nmbVYy$`h*doVP^fv6;ykA_R2Ecli3HrR;ypDLc)?ne{l1yWN&~2G%KleG#5k$uQ z#`%{0Xx)|$h*WW@k}Gd7_Rh-xh`}#mV1t2JA&e5XVT-it2q*6d+$V8ecD~p(BUBzo z=bx9LM3Ta~unPj;pAOI$NiccMLid=wKXtYh5=|$WV+lBOMic-@f+7p9cy^1_i}k1P zqVy`|o#=OYSE+^FnKe^Qf6GH_H(A7`Hz`dG2ulPHFG4zni&c8sp?1?xJeu6^Cm*#t zKzLmS>rg&5fljBYJW0|cnZ!od>_a$$L`l^;)K7na&1#G~UuA-^vb{NHkXEJf%qZkJ ztIOs7F%USbbdnv+SDTrDv5!#P`r!S{&$-DILAbg(f}Iixe{bQJV4|`?K=GgtqYR)7 zC38+l`<3kJ5YacDbmOcRHykqYV|ztdi}#JEt5q4J@gD-7y;`G&6W_}Qh@PuI$gE6S z8m!~4*d>Lg1I)?TWGHm6l9t;P{!FeciS5pK7cuoxO;5|?cOQhu87}2kPlALmW;QB~AI0()mG)j(5v&A5?k@iSEL-lj`m*)Re%)0Shz~?VfTSh|#r;=}aYT zOC+;w`Z`jf{4qJeg;zWI@%eVJ-SSw6a9u!(~UcNhDqQ$QknH5Vq25x31 z(D{L;q#n7^U)t|-IcE^SJi^2L20Zoc{ztS-;W1le42Y?`T%w;1EIpFSEA;gEaNsCc z9nQIl8B$BCdKN^H3jH&nIKsS|*J{w^Ah$IaFVusWOT%Qhpto1!JkWyGm$O%Im~3p!lQRFRQK zK=`{Do{c-EVvrK^8H%Gg0CT-} zuN)ia|Bo%rwClGYN@SJl?bvCRP2YTj6dV}2f9P;NCYq!jWaE1*O3ez6z>y8&317Bs z7sF_i0^^y(hfpi>YGT&Wy=sX`;(kii4_ZaQMfoKQ{2^#Rp}mK>bCJrw6(yq?LqT zQPBH)Syv78@e|R!S3v|1gUlm(RFS?r5rPaVIVANXi~rHE2Yy zlKx0p;gnjwT9VZ44VENKYgY|p%??M)-<-}ir&EW_ps5L^lA5U+K)zXjoP+imlGvSv zPBI)WzWL!ZvJNS3(0g;IvTZBqz&fm9WO#ITV}VWmLTsA_{UkGMlf>$yio^q>b4hE! z$&zo1P=x9RKkIW@Mn4$~3shFzP3qjmihT6p_?L<4O??JQ{76>==)ef}+8yz=aF}Tt zp8af_`Pht&%b*pA*_s49+I@$gyMtiUpCiZv6OAYuPso>){XKEkIG+kZ!Yy{yMUG6G zIV;zkc*@r|u6si8oSnu&`qUUhzi=7o>oK(6Q<#?H)(0SeOb7|^gk=XhsoVGDd#I~p zVU><@o`I5hxCK{z!y!!nLR=`lQuhNc;VR+15>K2#u_`VPS1Otd6nh!9&HnWJRc8qu zaa!&PpSP?uGlt1OXDZM3C@&hI5NiFn@^drxm-FC9OIzgnqh%4#Az-37TgTLdeV!qu zqcS?=1j=swfJS=2wBu1h%0c9!X()_xNf}OOKf^Peb?D} zm92o5^g1gFCbpk*QsnoY$(~T+6yNTMB|LB6$!2`N0U{3qw?2FIOg6-7j%sc$ForZL zI~zj!?5*9521c#}sGn`;Zp^TsHf1O1U(PWT0-|40)?$LLzV)aihElfOo@lF%e!t>c zrWz>;nHR9tpMVhco7qD3`x$$~`oBObB?ItmFGO&$Y-fj(=l5jq01LXtof#d|xyIj4rXq4Z zQTeo;5pGH{{2YO2^w6eXG5E5m=eNO#zylVZiyYhQN`2-x4#3N_Yaa8ai|OOb1&f9E zNBJKM)=Kug{yw)U;*lN= z>prqiGa3$XfO)K#>f~|w0R4&w2=CDuXRAzTJi889H(pPPNxg0(Tg#qZa8j-9SYz*8)SZdHj68=DOsY~)D95hO`OtN zdL!a+S8i8JGrIsv4!gtR}HjbfDzH z0e`A}{ebn^X1BY>R5pn__7zuV$rP@saq8^~>y3Vud-l<05#o3_116-&gF{7 z6NB+&?&w!RA0DY#uEhX6`*^zImY=u_*z54b$&J~dlh`Zr)5G$Cw*%!=pUmCl2yn;V zq_q%U0?;!+Nz*BdP(!b?td!jEPaxnY{8bDj_6WT0)q;}d*O_DGMB?rfMtM%}Qlf!J z9W(cPL6zg)$WXbmHOY31f1zGIr=P^&dVRb-aO1o`EU4KNQ4+mF_|H8P3q5Ob7)uvc zjOqqt79L*f6hPr+l1YH2+_*S)t#KfjK9b)F?7G*mpCj9EzB`Ql6w}YhSrz$|jDpuh z@!N&qgbPFlM(hLWlUjh|hNLNfxV?atN@5$D2`#Bzd#zR{v>h@HKd5AYb3?dW^8GrG z@Lc3&(V_4!jsI!>3BzD)0VMU40QL$^0E`;x4~l_Y0~W zYY}e-=vz>C%8V`l_|Z8O%2L9c87+?lXys6iFlbLoJYrq?D;GB&>Z7RqX}}J^HHJL!#02NIjpTs+|L8CL&f6v zcCWdj(?85s15kp;E+LL_D{oy{x0q;#ltO=u&uD=0Si-)j(QM7`a>kDS?{E!Jq2m$+ z|8O-#{zPlRr-vhtp}6ccy+=BCoX>8@buAB|cCLo$*b-kTBTr8z6GkLdvJ|*C=i5l=H=>Ti>#S0&UrT;Zv>K3D z$pfn${C1&VDG_^>`q|pQJ8h*Hz`q2`_`OCOj3gdE0W9PQ6TPBo-p*{HN`(Lv$*@eF z^~Sp@<0;AQ1|ZfsP{t)SZjETim0HjiWe2ZHByf+J2j`if9Il^^+k}6X1Dl4HkS*>ha#R_>*RUHZMYP-t%`{H|mpwGOp zmhT*eP7X|B3jnhomC=@?_YbQ91l{M+s<&GYcw`-rjhTB$F3fHb0MvuX%B z9pvMb{el_BRmO=M6$FUj+UUA=GODx3reQa~V67S7Y$>WC^zNW0?Z&=Q z#&$!TB1JENgmTShEnuz0=1nsfA7L}z<7zyk?!AJ&jD}-*%JE(TN&{yc^~e5xR5X8r zK>j_X76uOkB)q|&Y0)YcPJZ2u8mKndywKHCU{X|(_V*0T*mdqpKiXn8!?)t-+_ z6j`!bl;_v=;JeKBQ}vY3_x+5}LP9=erj}+b+~1rXuxbgD@;)~J(;GFjMzh7;&GjON58v^-SLuW_H<}^E z9{=MtiKR_Pf@*_((Gn(tO{!? zHv=4;bNm6S5eTKd>X_(nH~rUxbo=!MlArsl<0#f}9svQFNzxXBW;Z0kxIUP3Gv4!g zu+_l`=N7Im2RgXBM8EWhoFfo z{2?r#Vg-!f=Gp%0Gf3pLlNC3J_1!L+O0E>CdGT6@fX!5SQAOJ?)}8YeO>C%NjC%Jd zP$*&vq}S($@K}#}j&8;c$Zi8kz@;gTRxI6MJd<>6FzWCN|Ji<61sb8EXRcyog4z=u=Qd4>gN4*)TMBI7;Mu7tmA!v%yP{FfRIL zgyFtIpp6e^Uq1CPu{Wu|E-O&V07OfDJ3yk=RTpo32}ywSX^4Hr)Fxb0D89pI%CL9@ z0Eq_hOv8BXq+q)BZcO$AfDb}T^>}o$d1ke&_s7c+M+89~XB-o$SoR=3KS`>k7)K<) zAbOovX);rcb3F!HCCXkES`#H`ECnfEfrE?Hxf@kXqwTx`rh%0WTP}VtQR1!ihZFR%Q^mB$dD~nqdYAW zj?Y{E#z*&ODM7>A`?o~n(SgG51lm6<8&Z;11h0>;XZ0Y6Fvlq%-{w0=dg87lI@0+4BO46{!2=l3x%SrRdD8w1_M+05T>W5YF+9`y~{)o(szxSTw( zqihbWta~e`LWy_sb+RbliM45GO}2au%GaHWqU2yRaRTiAK?n<1EuY)%Y#av_Ss_b3 zp7Zv%SUw~)(x7I&EToP|Y=9r&?ZzTzItKKNy$$|V0Gf`Xpw)0W3TaSwQ_?A_Ncp`uodC9W;ZR%Dz zh>3{_kz?L-%jqs6x{Gu6h_@rMt1$MoY1Z53m9KBJ%Y#|&b=#xprzW&&6 zfkYiw5_KtGqW&r51NA1$M_MH+l1;TT)2w(BSuT>z{(u^V{R(v|`C>EE)GS@F&gZ+9 zJGJ?zl`f-v4#&9eN(ScB|*L8uXV;BGCuKd)`jgeE^ceE~hpljU5m4 znhOf(LZqrmH`9?hBA(q|9+*7hhHO9=T<-=NABwJvu5q)O&W;X;@sHr1*l}X;vBPfB z>TLA@#AV!uGI~RXI}0C`H`ycQBQ>se$ZF3TPQ}-e_`Y3nR8lGnU&6sgu{;rl&XVqH zIM1&GyzcivX2&c+y|D)uDfUyK$J1^>NuuSTi#Zg;asAyGD4!UWWLFM7Xm3L{DKvVo zX~Dd~uLGWY(3@BcApu80T+RMF;Qlkh6pycEiTqo5PGP3}#s6`3CH23{j`q_A#kr>7p-Zs@1VT z7x5M;G6T_seQ{*$Z$u%Hn2ZDEp`%>C$!+<@@Uk~6TVW!S$TQ+%LU9%X`HvGJ#CL92 z0=V^tw>FKkQLi}U3VB3?%MNN*mZ8LPVQ~>0z=#isG~yWo*BxG`WtK-f4C2!Al=MVU z^`)XxQpQ)cWr;)iH0$lF;;7n4QsP(L-~FL+?6d>V^_nrk=ntZBqG*hfXD-x{Eh|9G zSBuhR!l4OW$l<|Zzqmg7GfcX~B5aQ_lq}{hsF%H5HmF~TqtPI$G#sV+63Qs$P@tJ* zo-5yEG$_80rTCBlOh6gRhZ73xw`)Hvy34-Hs=E1Dpg^-f>pF70kbnOWNGlzYuULq- zt7w6qWHEBQ*I#(C7(Brvhi4MaWhN9x4cBg7k)QJ~zH0)S*h(!A|oibs!lN zIUgkO39)rXZwy~Z+2-5eU3a)hL}$4ESi&|x`@Ny}Gcg?LuHId%klVfMG1_qwv9%g2 ziLOAF&Hx83u*?0sO<%2`Cox}QKAI3*J%TdB;iH$`Yg>YrLV-4E+%Yy=Swm~UKwydu zOjrHg(FP*osGSZANc1VJSGbGECrBn(?9B{4fS>QOTEQrb5)%$}LSDtJ$|W4+IG#9( zX1vXK6B82IxXF5oM2ZDFgXe|Hh^_erl)ilD^$Oq6G=;ZEzx-q22Ra2xCFV;`iGvEY zWR(~y4KlV7>u;{6%?a8uG?Og+q$ryW!dWlNx{j<@k3JBS6dwUuhj%fv7el@#15Y0x zS7Pbp$pC%E7c3uaN&2|8b)*28yk=#jXE$=6KgY57=dP-s;A?G z2CvG{0>F@Svag_ex|LFp;F`cW2tF|m?8RtC%wwnsxTI;YJG?=gN5Ufsv)ySnZk7>@ z@!uolD84vH0(3Gf#RU))S56uE%Iw~dy6ZQV*fwd^JeC~qWjCulKHIDe4vOE#mM4TeFEhwf$OtxZB#cB`}tLICN)$)SB$g z6%>=05+iX3VEy1$1}YNlB(x^2;kvwTU=noL4eQn32_CP2=QzKq&JExQ=hxVpY28`U zak??n+cn%rQkJYH^Uo_RN8x9#OCiBqd0L$*X&{`g?PT!i6E)&rG25+z!ArDqJ1$RQ>_n{&9Qc>Sp_42mSJ-p6=nE@%zSyO(28i3~SY+tgWCg2o&dC>qbQ zq~U2=Ib6@sW5l3@ zXtIXXN#b;KgdGM@eEHfy@e;^fjy@nUV=jd`2l)F6SJT~4eI*|EJd#QL-)g0aHmukS z=Qv^&%k3Op9&J{8(4Is*GhHniO=m038%n2Q&Du?D*ikP-R&A~EPR)zpUr+ESU^d@4 zeJVB!aq+ojP4wT5G4W4GV_qpt7qCiGD?cewdlH>3SAAgE$T%r`;nxzyDxNe$-D%8f zXEa5l0Jo4cl*|&jCjRW(+p*({B~gj&0DI66`3O_8XFh4!HhLab4 zsYFY2!d=^Cd3@66Tgp&9N12TXy27YkaiH^`wV`GmT?%)@7TniX9oos;1zhXNw-n>i z=g#OSbFR?t_iuwl!yPyt zj^%JfS9U?z3JH+4M@yM0_hn6!6Bpz>OqOV^*b5mnl8AyUrM&KE^{#HjWhnAdM1%Lb zUxGkpr%vC{$>Eu(SpGry_L4%%@+S(MKWoBAj zpcn>B5F(illd8Vq5#vw(>&yey2>gM+8CYc6KD@ ze8^PO7c0q5E+nOQ8i?vtq+HlGzAd{5^5e`UE7FJjm11utr-;bW5K-Pan`bq3UW3dG zzio%5uMFUjdl965{+Ro5J#Vtmni!caXv#q^13ev$YE&a0j9Nt;MBOPSex-5e2%hK- zWmX&H3(mu);rz+jc|z-9 zL%2#@Wr+7looDosw;?voN|q7&6NsoXx?{O>BDp6gNy#bGl1KZ!_|eXC&vO+6goRJE zb_geBOX=>FcPC#6+h?S^u6e(UsM<^(hw#GT2}ZuKV(*kdALw)TW~`d#_32$GH5vt~ z2~CqG0Ai#j6Q4ZqaMJbsi;?k$d^+XW>w)l1XQJKm==xVq--TR0yjyd+Q2$$CNq!0J zo6Fs)0bCs8%40I|6qM%KAX2?8jyJ2;0C)Lf(+aSc3%jJ?7Y&P%kFEU_vfi0;q|!a6auw+y6;H*DpmY0 z!d!CC#Ok4?QFTZgt|DKCkxbRu7)Y3er~06&Eq4SF6+PFzhmPTJrbuxGWxv}3 z^j)V%A4Gm{$Ofr5WBi7o>?;6-DP`?Jpfx3Ud>#(l{n@c$sZ3&{*1XrAd8ge^SMzIM zj);6-J;j7l*l%}p7)2@i6XJ;T82D*%C?9% z`Kif%Ao#qG^H7TQ%A$iCBE`1NqW@yK+EeQUD}#&_N(vOz{tpZgTK)GChA zv(swT!TAXnBn=cUOo6S&=(&4*FZYK8T;a>MkeOpa zal7ttn!qxv3|R1&!#0^5a-wExNPIp5*qOKXXg}25X?KRDbP^c&hbZU4c7F6B`}Q6Oq|xQUO#)Z75ybW$J$$`qM2$@Rojm1!3?Sb5_QzzKVa6x? zZqv%Zukk5u;gZCif*et&!xvmYl4re{b}Wt}+0r}9jhWn4+npX!@;i>M{)~rxNd*KF zB8@nqeqb6b;Gsfz&B!2jhF?VKiQPx0A(*caC7; zg*%r_E(AQT$>AVfu_j532}_VAwpF&JBLgl{I=XmKt1E_}OH>Cxz=W4o8WT`#S)asA zX(;w1xgy~+;&Lt<41yS~-Gd`_LsJ;7L$Hm`LJ@jGR3kXPa&Z3SEIB2i2XNQ7HkCU+ zCx0Ga`~SFm>xZbewtrX=Wsn>|ItLg~X^?J)E&)N1k_PEiTBL^Vl9G^6q(MqV8Yu~t zM!Fm6_^vtk(Q|z6_j&$+_qRc3_FjAMwXbzu*C!Td`_lQzoA)1}r@>|XQtjfplzPR4 z-X{^f2;HW|SKc1-<+i4{<>WrrQ#zdxK&>ZX$VTy319y8~YU1QnB^Klr;Q=U6Zqo1V` zE)a@h@2zpiGf_fhmUF0X2SmlVb3Cj z$YUd=$OUh26seP#l5`-bkG)kG?oh1%C{VY%BH?B-mR%x>Rgok@)_A_~Hhk2qzF(!K z<;d#2c@Nt-#+HXi&B4kiianDDrYDW_Uv_3TANF-8>sNbj@iRB@tZjymn0ZF=VvMkj z_xINDIlGNsIG=KqZhe96T`FGdQUW;KumC8_KG9s}a^TRsbY}P#iYVB3v4ia(o9`;M zFv&Q~pB*J`UYnKDxEAwh4iO~d?Fsti!8txpRpw?%KNGMUHOT+i(wqlEwhufv--1s%PvhY>-s=NVH zWhQlcG1zgaAMfpV(~thgdDP$A1r3*MJ%cyMwj zj|&R0t#TqvQax-XgY)!T)&n(KvbTnOqXKy&s_{gGs448cHd`awE16xr)~CJ);1Pcy zHfQyE&5UE66aLyMkhS@fEj#Xq1JdY{iwd^ui)C1pn3M$(W{?-_#LBFsgWk~j5&j^- zu*Lx_!%&JjMRKEPxn@MmfV|7PrN3M<3EkkSnMzmR+?I2czrsdegwcMw#%Beee#{)x zwB7N{$bZE?7ua(eebQ8MxJmO1J{>N^ftqOWbuSpD&Vx(P?PrxF$aFJ7Djc=4lr!a# zI5#dE`?yRT@8k#sl`Bj*aVbC?@7m0r^)S^7EF606*O#@YO>?gB{|)AikfPsYkh2TF zDt(%(g3|l3RHbMNxM48d^7r7(m6_9yxw&oy9WeQ+{Ri646?275iI7R9lE{R3cP9%f zE}wKD?G3vjulO%MVACS){oL#F>bxVp6jx*SaR&=HwX_|JP7b$dB7u}gQjvR#l0AH!T`-oTIquB>m*@<*&G4Vr2{%x(y?ah?_C%=(ke7`gZk(EU3 z32&}VMLq9NLwxvcw_X``gpAwTH)Uf}m4^?v7#YPy7}YP${?(3+M3=j6L7#%k7t=r6 zHy*cp(I^n8#g$t0X9Q~PaZnvwsr>#nMW%QRF++;aohl<=&Wga+|8g3itO<+M=%99z(@h-5rV#K;40PU(_1ln$oV%$dmp8*~K|w_vf#7 z8MmrSvCg97Q~83@g2R1xz7*h=9E{b@0xOCo4ZERVHM5DtK_gWXNVRO?!*H|U!vfch zv3n;EqW%FuQ5fh3CXfBpgxe^5BQsWqWtrZiqHqahc4-HD-_if`&6}Ce(Yu;0-cW}z zUmc9UflEn>BK!y`H3HDa2pm4K`9U@UbDTNkcDeA^&1uO;l6jV^PW@T+JkqG?nP!Bw}ocO zMp6Ik_XG?-$vXMb0Fer7!fnioiICx8kv6p89S;!UiO(FO@aW_sy;`ob^TTPKqv+WU z8NX-$N{sf?3A91%n9A<<>12%2?hdz-;mtncX-*cZZns$Z3C365> zA(6$ZPg6$f%lw~@3~WTUu=wi?ehv7{DM{^5?*0o1{za$wJ*%$M(Mpul?zQJi|!$0yGQ)*H|o#N%n%d_B`e?LkMaD*)BpOgix7Wdn&D9RCj;frQ%QhRu|Pzq z{*N(7GocYM>;))B{2!;fDG5#`vAm-FFK)q~1IG6ghDOt-FzG#E`QLBPeSE+pb}Eg$ z^*??K{`8Abehl8eESnXlEc1kuEaTb)ic^==50`IlSqI!SyBqvSlfvep%qH||JgYwE zgSXm3^r*hvSXsYARjKrXVAeO9kG3}(Qh!ov-nAwBoKTny{Z9O4p<4@E^X|7Eg9DH4KzPs$O~VU$mqo!W`-_iAlbNq-cD z$Pb&b6wpWRTXcV$LE4$LocLa-c;5VaR})+Fa}t)6$7cM2v)PM?Ctq<-UB}ppj^;`# zGwm04%_I|U-ja$F#0VC?nfWwogCdx!x{$Eg+ru1n83rD$#YTe|hJjBP*%eQZH@_6ASWZ{EJ+tk5 zXtc9?3tZXcBR+`8*gf1NQ9+T7rO)W>q+bLYK0MY*KG_)rcj{}XS&Q$}w}dwi#zYLA zH3Li=O9AJlGhO`a9o>NITR!9pXUCdH055VDdzJjYK(E%R05C4J>s+jk)=IFoN5gyH z1BUs`kg6E_;zPEo8TZZm*$a?|&0NA?K@(VW&ocM=}pg<*9kvEu=a@11sTdD~^L z$nPbWUl&ggr&p*P)$3eV?iZ@R*1oA%HT;Xms|41MXd(SD=0)&0YJx_kxHK5sQx zy~-Fa2V|u;XBRkLef)k&wa9)h0J_Dm^;W)=Y9*AOC25jzPLqGV6Cm=4uLoJX3pFfT z?cbh^`e@PW_4eUynpDSh$_tC7vqbc+wBu!cC^Ltj>)N-#XuvU5L`pjO4cUZV@j8z%6h28=*xBVzP^Nd?}9+ZC6NTHS6)%hI+ z{|;0(254huo_o9BNB`G2K%0XKTm{|ly1-QXBhLK(vuTl$R>JB3XGgj3v6o?B@rQ)v zuK*^8fd=|H-l(}|dk00X(5`p0)h;)cPb_@*sOk=Cb-Wk@46$Y{UE5o}#TfrtBvK?m zEH&4&=vDtCRRb^3&4eqnvmkG2nW%Mc0ctC1AHazxq!Z}M^u4fJIss3)JgwqEp?a25 z3fIneVis*$2`&{WJXBpn>x_CdY1Z#j{^z3+_rnC<2BIh|aK;}`6p0lDiFmD;glqk+ z2oo6%AWCcy1tEYV*Xz_6dh2cwYxM~1;t`nkvr#|)v)=rl$JX;b|7movJ_;AV0Eqxx zkk@LE!KQX0G3!y)lPsIj@+_@)j}CnNZ{U=tl_(_K$&!g6*D)CS53kOzhaOzPUU9AY zf2|qEaB)+Hfha0L7l`M2S#U@cGA02qY@TKc-v8^a(N>_Yw`lF%=zm`?1plghMCVLP zFot^RuY5B4AsXYJ$Mdm&>fPT{_IazE_l^J5hP#U5(AZFNshOPn>QOSR*{5qRfQVqpOoJ z4SY_xOX9oILE{@P_U+m~Yw5@XbU;?32Dj=~^_0h2ZvMPCu{OA|562$_=tolU*)@E! z>+s=d{PKR<$GYPSyAUnlY<^!wE7x$!eg}B>UVFWKGhlk)9w~wI;kvF4%L}-Y(1ebI zNonRoD#WGX8NKI3PgTQDF)@SJvTQfl=B`3s|Mxo#BqW%zfyxigZ&{&%tF@~Ej5V~F zNd7Mq4wc>Uw@p+y{bgNR9YLCaA@l?!%pqc|hK*ilWz_$KfTld!Z4?QqqF;0zy$djj z-TgKCk~v$r_)&P}Hi)$;B`xOko*PvUNx4krmcay1Gy#-V5^C@L0SAzq1p>17uvadC z+JgXLnvN30;w-?s&_QqR7d>SOrxY0EQpD$k;-%iRCdL(9e&sN%CY!>MvVbn1F#*`3 zGg-mZH{U)i!A+w9o4)>Bg_l-ae`i|`D*+v-txrR_K`{(xM%4WcDV^rREL9Q5}e(Lf@>)2jT2yjq%hqFZgHI>tCYibfk=}b9)IfXa_ z)KY(G&De_wWH5dx?dRHM!psP_wzeN5K#0HHPGKJa+LX7FjI7iC&a+FqP**hH=_>ip zO9b?VauHXo1`t%BEUh43Ngh{2v2wqLG2A48#z5q=_f^Es_r@APOewZdBR>cv1%V(O27n&D7=DU;gxsoqoWW$NQOhnOPxk@~oZ1eR^f>}N7NsNprC<9qG@UO4nsm1YZ1NLxRw#<9L*ZA_@ zep>uoIY2AmJ9xi?7kg7NRv#LG$+}ZF>w>iG6u`ixwA|<{r8|kgCk$s^^j_hLv$QP!U3Z~A z+z)^hf%5YsHJpzty;Y({p9eGde>d=CJ=^J^sUE;|Muqoiq!@pFT=5&?BM159M+$7P@WK4#%w}*SMRY(E3N#rB{sCZWE_7| zJXbC#jnTq+OkQ8rYZ0`|yZ5g-+>-$tX6 zH1@$!i;Q=0n{3DDhpr>l8&>jCmDY26!7I@W4j>7lqfPj5{k*^-sU*vsnJ~hiK(h-j z0Lf^$0O@oxZFlVjaJ0~7Rc!rs&-z<((=s#(5tevcqy30>ks5fcz<)U5j;Uk_kGkDn z61c}d+LdOh+H<^{?Bb5i3n{W4tHea{O21B+itbB~HT=HrKycl0dRCMqyy8Z#JbdF| z&amF2FI&X^>9Txn3A+O?m~Ef{6Opxz^0i5Ws~a8w!G?GyKm1?#D8640{CSkTprels zpyXZkC6q#V0X3}ht9gNvdbvbqj$Kh`+))nl>9Fz{i0aP-tUmLjrO9f;3_PB^r;3j9 z#pkbumLU!M%=y;QH_uhl_y^AH+k-oiuZ~cB(wtj>gV)1iA>AV_mwOIQZ`(}c4AXx} zRW_Q2aW#&|6~+YR7=<8Rw%=8c-I)MOBs`ke`dg2e#~3V<3qBTmq5OcUKBl4&9~#$? z(Sz@T2h22@?3NvJ*wt6%du8_-VPAFNLIx$j@T5BYk^78_Nh!A;Q}c@ym8!po^W{MJ z`V<~Sn_f>66oy3bKl}Pd6YJUy?9RcjU^TXVo6S$n4{uNqXdbGT{At@PnOCx{xue1`qBl{C&vCm^z)u?i(&aB}2Lv?3OzSmj~&Ir|hZA+ZiQH<-{~ zvk3Ts8aQAEV;%on@hM-gsQcUfXH}XlR* z)vgMvHV?aDwktw`87u##EL&cmO+b@htw^h)p7weL$RkKZl%m4uu2OobM^0F=1!>5+cY@URrdQ&z z81y_Q&IzNfh)#^T=ALErg$iL(kaag+&BO&8aYc8sOO4(`UjHBV_4(Lv+hjO9K_Eg9 zmTE@y0KvQ}ca+)_b@_mpGVDZ;M-(a-c;y}rbP8~zNyl$d&_&2ez_7b!9UY-%Tm##R zP%^8QX#~kwwpFJO_C0zLSada8_3^3u2d3Af^jyajZO)e$Msq2Mxq4Eu08V=H*^Z@- zI@dK~u^wqF-LiXnc&OV2Kma4h2^6}yYmrpz_byMBo{Z-U7TeF?QLQ#`1? z7KspLTHL)nx?6N?t$*Y$eW6H8lsVe|q}1oY*3!T?jKg-Se3ob3E22uhcF^eOOV^qA zn$}#iCyfU)u@1Wz8!cE{7rN^7hF?MlB&;tU(fAlrz5SSW_44Vl?%7KD$Cd-zy{CQ* za`2gNH>_X;tGzIdf3)M>gXfJN>B2|pNt zX(mUyd&s9TeT)ZtNbH!`M6`P&b@*n9Zz;c0axL>liS5rj3sS*;ta~D=sF^=A%%GUi zbk&4U1YHVUc2K(R=@#`?mi=_2-&hRk^;Sc$%HxFk>}BMeZj%m_IWPIPY~QX{o+M9W zrj*ip$t}J9Tw0J^WL1+COn1W!Rnlpi)Df$tGN(ddgJPha9GZJr2EovRH z0rjKoKje>TJ>o=$*`P}i-Me6K{t_#9`llmRLl(&g7FW+=e+|IpGXhlLF0+0WS$`^ zVddDtSRIPS43(iD9n*7Ee0DO`cF8zYGp+M_`2=G2J-?RNpY*bxNOq-|b4g%Z^? zazW)R&>;S2hVZYW^wiL`h8y(RFi|*}FeD+DkAAl|Lu9<%^Rd6UfcJ?@3qsJPBa*6D z)f)wq=H`B@!-_>FRWd3Ne5{cQ#&z2~EdqovSB`H(TdXAuWr#1)4j)UAhzFXMpL-%BFh zl-tKpIGJlc4U|VSp#0>{d=E?qQIgKYThH<36?n)*Q2F@?$j0AId?8d7Hj*?0{*I{R zWmeMCE(!heC9uPxUy^|n%)%S*Eij;9uPwak)OGsSq05WIk46BN&>APWJdTm2F5A;M zP3oa1F?JU6wc8rP^AQWuR$vfVN6w<`%9LprM3S3IOZyZvf2A67PeEC!(a>BJT|SC^ z_tkLuCoAWCe@z{&*h#y(l{|;ud*73<`47DZWp@xtH);cVkvk?11$AvVE#fjFS~^Zi z?|$2-QoerZ^N0S77Ht&ziUog^FV~b%{u#ifQt!*`!$g^M_>Yp;cf+AOhoETE_Kaja z#kfp7Kn9wwW0P^Uo@ux(iUL_jGw1_ZZs~jf{5k_*5^hf1fw*#3~Y+k?{ZUVm1yst@HpJmFA(Js72A_wUZ89V z#>1nR2`}x)wefr)AOTSY=9;7&kHz0;JQVu-!FE1FW8}^d553~>&FgitrZGZ>Pb(zn z`nwNPxVS`C+(3<{&A(AORKC7i`}os~+!Y2$a!9??sk_U+rFYSz=)%VRB<(poou4Im za{D;dJdx%=?i6#Bj39=o$A0pUG4KW*1sztzNsGN1lsYTIV;N`%y_d(T-c~J~zb&r2 zlLl!2aP&c@QBh=!4G*`=@_j;l?>!13=0IO$P%hq?dHXbL^#n^6Nw> zP|q-Niy|KKu1YYQHUc#)7U(263g7LrRuN?1Lhc<|3r9MIi!MIPpbrHlUxpiYeGFm1 zHb@^|YAb3HNwO5qBRuNd#YAuTPHvr_T?h-pVN$>QQ<&&PbSbl%c^No&tA|VIUO+oO zML^^Y=@`!g(~so*W?Z!#91ZYUM+de82vtgX$OnZTPL_jTjPM)hzWKoqtcFIb36x(d z?+g2g{)^NB5COPf2qDv$%pIGlSWCn{O3h2W(wWBth#4GZ(JWL??%GxK@;Y+Hnn3$J zfH)_^b5lT3HUqey9KWkyFIS{Je+lU-U{}zg^D^drn&q2)$4Y9SVU})5wGCeoSX3N6 z!;wLV@b*jeFl@Yhw86IO?kk*cX9oI*Jr)p=Dww+$3EL;!W=CHh9Q=Ut+#*9T7P|G)_4k~;^18v-W{W9O?n6Ft zw2}(-fRxn6B)7jntu914_()_XRbOWMyUE`ZOimW>rdhCcryEg875;4WOFs|L@Tm-!K1TIohcm? z0@RhaYW`Bc;J?2yE66-t8+tCsL&t|M+*(>tkQ@$^WPA_wKrK^!g(BH>SX)SoTbK{ zi22&89Z0R;*`voFQBy2@iTYjjggjTT)~FdH(RYav(9hMsFU!Pv3y7~&Kkx(6+MLKB zu0^n>R`@=ymZFYP+p}}j>Xo3Sl_-Gs_8qA$KK$1_`ww0jO@n5-InTfbKad}PFizhz z;QIinxNdJEIz7B}xW_oxVx1b+(r!w*ksA9T_q)c;=XJj$8Z+}%X}Lwo=sJ|YM`YC- zgAaALMpT3Y_OVD9fcI(Yf~~>fp@SAOK6eo}=&^h+72gw$K@v%Zm;_|fuc-Lj2@0?p zv`P$YDQ>z?n|iAe#zC}w-qT9%jt9D2fG*7RXGQ3PPxcw6YI##pzl40Wd?2Da(;;T; z2PHTLdE^S^cm{RxDRcz9ma~G3?C@-l(DEB`3>UQln8-Tm|JAf{1WF}#dFK4qz2y{{ zIb88cj;T<)EL-H;FUOGI5H3I`L8@4j{~&(6IS`9=bs%%EKDdYtUDgia0~6mB>abqi z6AF5n5}z9TtM2jvZKxhUO2bZ~*X>7K@YjMV8^9Pqi|7>CxL^Z+8L)p`FdhxH0SL9* zt%Ry0bVRG0BV{?55yM$oNB0a@hhu9u>(+{HEOf@uE62}22QuU~EF&hNeBkZz$7+c0 z_W%?BjKgDpIeu^#v--!P%&Y7DqX=ZL0A@z?uhB?A3W7{=_Y0z$6<{(uHf~#r<_KW} zpb6EI#faI-OSR+ji>0day*Rc|9Nh$#PunN4D6gJ|KJ2%#;C59+I8AvU>J!4D?4w<& z#uNYbxWKb_Gq*T&fKi{zgz`!NspFm4bv8Q`ZYHb|{sTLk0ka9EFKtm||M1|;%<{9p zF2N6*Y5TI9A3_UI2150DLijxCR%U7@fhNJiPdj-Cn|lo%?tCJwv}xWtmf>3Cg1ZEP+coohGq z-+ zQQ%osh#jqZwlmA&yx6l1NSA?R+~y3tL(aeAXbMPco(!U}tI4|mnL6fkxcKeUBowU% zjqe33s?g{YyY#BhnEk~CJ_a}=*T;F633GaJWeUov0y<&!rJ(PQ)U{u99)#tf9}b*(+QjA@OcrHoqG<DX-fqPhFw@v74Cne(!&s-y%%o5x= z3+WHpRcaR=>+1k^Zr|bm^qgPS;CHTC5hbZvr;$ltk%%3p)oQF&lcUII@o3>}k;s|C z>tp0gdldfl`0BMA0ifU8o3{Lb@**6q=pRhj#t998Hbu7B{&-UFq&Gu^>Fe<4yjM@@ zexO)6epIm`S4?9PJb8n&?2NO^fFyX-BK6YWzIBm{QtI6<-_$euO`2oJN~ZI~v9L;u zt4XJziVCnUu5Cw`43IM@0l;E`?)zffkY1sxs?!;Ls@bDNiu%4`| z63NdK+^Yq8^$pjHPY+uwq0K*FeobCXW2ex{q_duG-$W>3^Hoemby~AF&oPGa;hwEDZx!z)D)$ew; zBOGvs_zgc`lL^(w&OIo)BMS|FNl`dX8Y%MA*8!adB<9Z5ARf+PXXRP_ zMxux`(I8d;Qf#m#XeQm!>D+!b(E|Vq`<8>cu%mmXq0v;5LUxFm55Vvp{84_SxC~n3?7)heagp7z2gv`Go9I1a%K46zD?3WE*$sSw8#bC+91D6J zZtn&#ns~1<`*cUJ!)SuAtn;|Wh*D7~a!+7KaAO2L*u{?bnW>7X*rG6F;tij6GK-PE zzDiZ|H2>;wlCX@-@7{VxzK}G~%q+F_x@K(`jx#kZZTd+lW#AArj4AsBmeTl#fR(gdq{|?EtQKhgl)|oEO1oa#g8L1xqy@zu z)AH#&UHB>Bycp16*JyR^%dZt4-~!BM^$>k*97chwlyY`<{EeeuqjPbrq#C03!dg03 zxuQaf8)ifA_Na-i7UbC68b2NWZS?mXVsi%#L8N|3o|A^%Oz~o5c_2p$RFAx*H%6%t zx81^|uMu4DjQloigP0HmBW`XDA>1IP2F`2a)Pzv?F{^wX(*I*ozNDjVc=6c9w;Xw&JIdKxj=Y!Y-b0+Q996h5u#@_>Y{nJAuAD|v35^-^zF<|0RFm45W-)*2*dFvejAME31o%p!HM2U8fF!0D z?=ZqE`5L2thS#qAo~(SCjJAsuQbyZ!=}EYYTdQR;1~dmGd||TX9xwT!#mHEe(o^>a zv0sIwxFh|2o9psvbg(H0)5fD&EXA-+Oi`#4KjGArhLDpWykv_LH-H^>v}=rf4K_)c zdPS{laPo?E$>L`sQd|czOxJw87{m%%VuQL4!;S&}Dn3dDAknaJjvzw?TS+eobQa=% zeWLJ4tW5Zm;oA)3{^U`)H{zPZqE~>+3hqEG#LP}HiS6Drga$$vM?OgzMt1{l$1sRtSF^S9D2!>Ilz&$y@wpRbGwUjD3EXu+U%-a7LXp?!G(59*G^}K~!V@bu`R($t`1j>TvJW+iipE z=TC$zpN4?t)NgPuHK_?DJ0_#R6W)IQ^_JIqS%-~3=_;B5&@$|Fwy(>Cs7lqb2aUAL zFoWVbRF*w9NJ&O6dxjpS4)cPZ2OkBFgfMqlr+yYao+qSWUjqCc9d&4RX*(KLxO=&k zQue(N+o@}WBedjowGM0jw=ofUIBJQclDXVL^f7M`N^tHIRYuYmYYHDHkQWRDi4R%RvKilMeg(JY;Ost+uyvH%hnepPf|02uI6O~f=vyYekJ=VAi#?_6_7C6^PosJ@|2&Vk zeHD!gHqlNg<&q`JIT-IQJ1zS|>WAx`K;ZxYG)`j8RUYwu$J>KwsFy*Jqas2nF;2=S zZaiq*I`s%#8wV+!JA+-QtnF=tiVkzOVMy^$s?oTgufPADgT5*tWR9i{q5qTwGeizY z8<{00NDbaR+ReMdcm4G-fg62#3cJhL&O#rFz-`Q7ebd8vOI@)b&&DSCdvuIr51;CY zD48QYE19SQ%VxZO=pm@WmmPcjcR1M6B*I0X_6y&J7dB=09S2aT6t&~Lt)y)E$>18erZg7vKHo63YAAB8$N6R$x4~j! zCy$WIZ%rz=;=!0JK8s+#1;g${I`0Nsl>GBgEY@f)s;ssmgC1Ie0OP_t^zHbZW^OvIfPh6d`yh@gCV+A17L9it1M2J9(?Lgq~7D2s2O&#q58wBUTZp%zEZw|?Uo zC`&s^2*bgi#k%7XgG(J_k57#j&4!b z?9_K#E9VU+jr~mo6IY|ZPYYy?axh{??SktqBmyy>5M3|7S<

q+E!98_6k1o@^Q8 znZ*#2Y1b&}cqu=TNgheXyO<>Gev!hfLdufN;x{d4?!n2kX93NCIJB@SmZ`aiF;p$HoIxMEvt5SqnM0cBU)TfNnri;= zZ5ds+6!Q*q;<%7kbKYFZ5b~xi^9mwf86yjG;&2-Gd8lrW`OGsLrQz1#d^gy6DIqUK zVX1=(t0HmX4hd8lYiJv5=lfEBroAYS46a&wDVe{x$XjuqoC8`NiS8ARMn07|9PlpORf% zI()2(9Rg1y^}lnKt)<@1TmZc28yA=6&0cqt*mYk;+4!M%EvG<9pt;)erd2z@$hb5? zGC+`$Op737+2lH@1twDnyY-;Qu@loOzT+Nf@-+7Eb zC{a@_utCftD(QRC)OsUK9MN=A+BjDMqf6j6^3Cw3kt-SJt>A02 zAp+`3FJFA+zYV~pDrWIbpzC{rD_72}Bxblz-4RpU9A<)~Y@KP+?uB2^h_|DJw}%%5 zYxuuHLb{4W`31ulK4H*+E5>oonZn*P*CYuJcfm_G;eS)KJ%~#edmQ6~)&lHhMn@dJ z@cU*zLnen>_S~C4p<}2}R~&Fa+($=kh&9-Ejl2Ex8S`-8spt*s=*7!z0g2MZ#Laff zW8cr0jOc4Fe2ZPP(r`W9;@5`brdSdyZtrwn&5Z4e%$waCw2d4%KB`)%NUL!mu}#8_ z+QF%5EcAJp-;#DOI9vWoCXufl$}4ha0mo7i^ViK>;xF(Tbz z7xLerdOBe=s%A#JSUlk7@0rB@FNQ8S=j?@EIiONlyjS8@=ou^@M4Z>wMV~9 z{}Iup-OSX_Gwzr=mgP3-VYG2l^JXa1R{mBy~SCUIcD^;o-n))v7;uGwg!~60P>s@`EJ5^4BFG3y;h=%RX5p!}n ztqE#M_2UNCCqE=w(kpf;yl-avttX%%yy6SGJR%Q^t)|JRvfQH{G)*IcP}IbCC8Z`|u!~rbS>j7z@n|6nIJnQZ_SZki^Cox()ev z?J0_F4jKH3G#}Dv)S@|MBQ&jH%$xx_T5mYD>F<}vV(F&cs*%Rw%{BHEQC*#02VKz} z_;BVwiL4Oy|~bRZ*3qa7_$}f z>R4HK=OT1SBpi4ZB?ihmlKUF8e~$7(w^UEylBkqyms2G6bn-tQiaM*_4*n+RW!)FRB6~HVlQ0$^5M&@AA93`>9nNAeErne_o!)~0V zHygGbx=be=-?zt~j=1iAW)L9S#o!)g}bd}n0a;d8Z^0LMV8O@k5BCh5LKAIK*j?H zFDozOL%;37l|9g~Msnw>p3p&Bx+UB%C+6(Ad5@gvwO{l@uuk+mN4~K8Q3F_8fB#Fk zyYO1r!p!igMbNFcm*+P5d#$IN6D;fB>WuhLr*5wZ_a05Or+*ci5H_-V<~ucbWW93o z3Q#w)JcHpzyQhXhQN-sIKhuhx%?wcVjg~_zk{|;yy7YXq%yH!oSGuZy%rMvM7^Xt7f_x@N#1M&dNGe8f zG$g!a5ht9G#}psd~FA0zo`^B7GrvPI5XB6q%yi&zu`QNoqDTy`m> zKQ`sa>JZL7)4~4YmJ08s{TasnF@uqj)+4b`QB_Jc)q1kiHQ^tZR_we!Zqvupl&5U# zEMBrT-_10;>{I-1-E*i4_r`d==BgCzt~MP*A<7=Mb@XW1rANi9B^X8+LLiX)v_dW; zedq9{veUMNT-)L$?=0i=Ko|DMNd3kSB4IPR0u@ckN0*GN_8x|i2$n(T?mq3KemlQl z>8Ez3k14}xrn%~O;6fPFU1wV2L;*~a^b~!xkh)Jj?mrIq-bd|i^G!z_6KJlyc>NK? z)uApJ6;v1#zX{_65OX)q&@~XJd3Z0lA?=-gkMZxh4K3EidCSDJ>`A!vVWnF7di>H# zNQa1LVa9gB1D|)2aEd*aH~FFXGC@&`#QVL?lx$BkW;1@(8O9IP7Gx!KZ^}-RQ{6q& zqD@x_LOL|WDdsRp2VR7zh*ye;spv?w<5N2h6QoyKo;E)VALCuDE(}qj#kQZW7g6{>n=W4~;lDe(d<+&R(Kr@@9X>8{cXd6nx3v+e{an5| zNZiA=@`=dXeOr6!UD;IALl^pbX-jcKBTyADKh}IV4^?#DTUUwuHFk8!@OkV;Sr=kNZF~)X{X&!CJYJ1?>X{*O}untE_fLb%d&&b-IdTu zW0KD}`up_4!y8p!;s`lvyh^{&3CYWVN;3A57a~N1mK}dT&)=PbzduAzp*ua?xXIzG z9`eCKSV&f5Xn_8>AWBM~9Cpi6cFZBW%!FDk1mXm-4oQ8>P|Z{o&?$zvKbT%CtUTt(UfS9M4Lu`IncON z)fH57vy$&R8T!>a0Dx+=Ji2zc#Y}RbpbhJna?bAD3Y_iFbwN3G?h|jmg#A z8$)nrDI#w{Lo#>+kQxAsL_A4Ky;Su>SL*?=_$WYJlnBiqyD#4NBMF*Mt z4C({c!UQ`Zbd~BhLIcmzb$uduWCw*D^G0V<$7I<&)W?}L2KDy3Re#yE0QLm$)w3oI z6@fNhZoXz8g*MHY;p)fy5LWa@(1yJC&}trzp0r&04EI+a1vpX>~4sN#SwTD98+p4D`25d9A| z?U&?i>mD5f$YI6sUEsp8)ILHPglq=Q_i48ZlMcI^`h>4n)Y1`h>O_<#9rYlHeuoK# z7z+ch6IOzODWx5n04cY7ZOPpAQQTqyEvSw-a3pQzTb#MayPAc60LWKS{T$KQbcI%8Exxr^MxVM;U{|IrX-d&?&7c|>}DssvD zJ9~iADx%pAY&`vqePtG5YP_yMIcYRtF#N0M{uiq5hz5uscqq<7LQ@y3BC}8QzcfSQ zoOJgsUofO<az$J`zI9DmIE+sN`4O1?5dW)h{+&zrd5a1MI8*A3oj;{*!WZQtqsI~;}7)3#~u|) zWtNYWMmO_8KI@6vp`+wt|4TqbZipBLe67C0rzp3~dLAdi%%tf(@)vC9oNh1)AfmTL zV|~GxCZdplFY-AgOD`C7>P4sn(-PA$_t3t3WH6{qmW*n$FPzrCZv)+HfmCNv@IdRjsv|J zfX~`4j1)x;*i=ZC}xpLFNe-Gysw_mBE|q%V#$+GDcV!LW<-hK3Nh*TzB&6 zlh|iCk3A87+;qjdTP%DBB-ZaIj|+t4b$P9fSdi{%VSBm8;WJ>jF0L$MH;(=E&=rgr zl!Fu;yZkL*$?-9N2iWDs?Cm18Sr;iOVHHzOH4u?KVqh-O7>aw~d}3aeZmEH;n~AlQ zKgG1N4tB?t0y_kKaJ_C=(*kkiev)fU__OnL!UUNY9&_PkAS`<&&-|agJN_fJdv!!W zTkv8U>sJ`>#PgFJ9}dDXm5YLN5tLy$MnXO!=#67sEV8LWlId~A$2R~U(8!SmhvI6t zG;ib3}D?(!&ZGCUv>WV?0&|rpA{Kw~n|>;oMKB8dBU3 zpEhnSd~n~Ia^5^n7aHpG?W^Vc9ww|NAFb8jur|RPv{#MoEMHo$E;;Gca$4CvPD#~;$M-PS9BfYN zx^8o49`|rk^|r?7p}l3(nOuxjsS2_HW2lsUtmS8VVG^qoaU9jx%GkI;H>~mZ*!~)1 zr7CSw%xANgg55w0Z0OX|(-W^$3$QF9vEr%z*Y&Hh)`xddm-0o_Pv?4U?Zphm^ilh& zV5p8l+H-X}@qZ1{&+VpZ$U&>w*hc&pYt$){6;S6chCT!PTdQh*?EmYA{Jt}Pf8bzy ztT4wPNMZ}^oE(0-IWd~JxytnG?*01mV+l+M1Ie%he{wAms(1DI$B)Ndlf%;|-`*!G zveo9+3H=BbJDaJs(VG0vZU6N{)bOa#Djt#OTCSrWKjFC=aZ}@iT!-nknos;h$8`nWI#s)ODgu|5{g4nzHrbFjjx=`oS zL_iD3+w0!leERV;xy1XghyTYRIjWwEJ3XPvJSb}d8+Hj!)%V1sWLn)oaQC*peaRoR zay0!Gj${r$cDlI=fOKXpDFYKV=l*^1=y|tkqtoE=y(hYlfsAG$rQgfk_o@2Ue?1E< z@g`+k;wD;t^ncHezYk{%#h0r#%l#Sm$D#i^MiUx7z8u?g2?xj@ga6Mc#$dG3Hr;Y- z|MfthU-7$6OB=0s#s3HXe+>o9T4{0G(ZrZ<*x0FzpBTHhzWbQEhRwglVV4-CpWGO?`{{!(>o<&;2 z8{sEILFHRbS#9GXR@BG4ucZ#7HU@W_nbP$SXat3O!Pm^`yMww<61uyql0H7(aWlRl z>$6%nuv@vuo7kOkc|^8e#m`5j$94a}=3~{)CoBevq0yM8kn1;Y-1rO#UEM&buF|jr zn>IEz&BeySNf2h&8N7@uNn$rN`1u~kBvr%N!J(9VJcsq@I_=W-_O`>Uh7`>J|l_H`xJXM6N??|2E6AkEL}hBD5MmRf8^F! zQrP|KcX@`~#Eas|k^*Po-VbFk+VU)t@YBPMJ!*dOJr(}5p}2NBP9z%YfH{K`N$l#v z551k&wJJv{aqdtWNuT93u?zwaN%Pcad(Z5o@Xjk=7=`r30~{jx;g-D1y8R# zr=%6XEAA=y=4a3)Dau~N%L65;jmVWYJ8MhsP=H4NK;T1^jTVGG^0{^Asf5IZGoAiBn5$m=;WMXa_ER*AEFE2Cp^ zOg1JWQ^$TA?=QN#jd2-^{VMmZ5>~7~h$8-(WXsaglchN`a!b6Z)5*)FAaXt2$5C9O0e|V}|NGz7ob1yW zJ^ocVK79zAyq6JrLRSkog;KFJw97 zO}4&jX*sEf^U-8=m3oD3`{R1SHdr{0M9Dy&gX3%e@0u(NvF zcPw(@K#aSAi|z`!d!L}KlNzo_!N|G+$_h>4%+@vAV`!f+J`LNL z+52vmuo2iKMBUcaqx9LhBi`%)upP>pS64MbFQ_lTK$M=n=2T1OEks^3_&wm?nR#!!mz zpfQyqJ-~*zd!AwAww*H(Tv){?5^M1VwQkolTWL~1FHnsWI05xemBwGS^4kaPcF=2E z3A4Sc*@`%DrX=D%YOeOo7HoxIn(5V*GVr-!193PR89ZND?b#(;aI}0+?_BPXeCduH z0X9YjL6*QOmgK(!{J&m7{Ik7C=|mD>UlZeuHOZCpO(%n6E!1>Y$Fx3W# zw>75u(^U!%0vn4Bw@-_E7m9$1sX;?It(}7bLRp+8E^CR{sfBpAP#O?bN|^n;<>0M0 zCGS`9`85c!!NcT;M-&}H27fz9p^FBcbqdloHNrb^&G-{6$rZ$Gn)H8I=l1I znGC~wP3O#<_I>WR{pq#n{SsLp*De$8&9=Sl;X>J^Y0Q!toGDLW6%YQcLA4<5GK=!w z;a^2MR4PU=5HnZ1yCD{(pX;`JteZ?iNT=FHYJhDk5lUMMQh^2D+4cR>QoDwJybq@i z(EoRx7U299?_L2$&-=BNO9Hr5xP&)=&2EHCOU{5JV(vHEY4*!6zcfHwvu0gwUj1cI z3`RWP;oaFV17ud|3R8_2UHIm}O)&w6H4|aunGOh)TNRx>W;;FSQx%%%%7VB^{+UO) zLkn}?nN;47metQ6?zkPunQz8bcMbz5dQMa+@yAd)oFU9`Qcy!;MbMagyj_!MQH9~` z87vH8N}^`S0|Q4zCSurk8{|2((=*B0W9lO8mh0xt)7R#u>SBsQde@0sMjm5}3?eRh zxsuCJ6FKJeth}#FN~LJC&67z8)@QuF*zOVh*+bc^*SoY-h}|ok;P@@J)xr-aUC0ES zQhXx&*jDFBc;Ep*yh}!hjyN_-V&)qrMqIQiHf8dk=D?Ep2DatdnQ@sIZdU&Sid!hn>tW2({1aIq z$c38vF>~ws)O5$ zw}*|kpYYrhKViHyWz5_i_I7szW3h=550{;qvY2H)=}r}1OEX@D^vUg?WZu#pwgt#x zMJ0XNpE3)1i>YOH=Tn79PT~$&RqV;eAlDun)s%#2I3L%;4Y9TJW`i1D@*Cvv1^GR)kzC9{q5&F;1e3s*zW)Q}()TLeg zYL=$bY6vdrR(6~rizHo*p;K)mb&MqrqcBm-#fnwP9V$%pn!Dz8KXHxAcTPO@ycUeP zNQUM5BvJ18^*k|ok|uF1cWsvoM4IBd_WEuvl?*@6<7W4_@(ft{h7q))^j)HcK*r{9SI zWeWMsP^wiLZbAZgI;?j?SZ0vy5x2`_h4Z%yb9kOmc{%TPQ!MS4*zFc|7yfwM6#L$y z^4(6e-sE1{Bd$YXZ#QEsN7*lQHH$a#EsZpD^KU;QD3ik3f9cB>VAI667=BMcja=m3 z5oO<*NUnqQ8P+F+V0C+q!rs1hBICbK;J1flZrUwiIv}o^qt%sF6<(Ipr?oVx)#$bO z+{@IVKT?@)x&Ahv^Kk5_z|Uc<*)T7~U4*^!K3cOrW|=8PQ6|`@;!m4O1WS1{1c$?) zfkNo!btL)1orN@mR^yO&0#)b7lKuEq>qa8bwENGeleaI7-<(q1dx>XUDSk?kf%)K1 zNm_jQqZVaiQX#4H{cLQJM=LqCs0kPS*I<~e%Jq8-x=;J8HtvtU_fF&64yUo}g{)k2 z$N)NTT-FewG-2K_4mmqN;Kdy{Pnc$KdKhNtA0``(YAbkSwgxY4=7m6?RDm< zP4O(xDSz#t^H2&!lX@0@(@H0`BP?eh6EeKgm|0(pzkwJba~x64yk-F+s@vdAVyyO5 z6)V(hr@6UT_mZ(r9TLz!)4-Xl)2i;QuC6}I(@K542zQPV&(Am}N@YMn3bRjiDZn>` zPmKS&Y~{cMu1U0s0;p9@+NjmPLzN#Tg=8u#d8QFZ{= zH>MN;)Xg8iG@q3;i@xY03 z2w~o?BD55vW2j1?$o=qY>Y1i6mDl0PR=d(MGpG4a3`|Ct(QYJB7MXnuWZIzzl!WF^KKUXtRV?+Knt-pf|IQc<0ZDMN-S1f_HrS=X0lz340{ ztFSajz4~D<&TFrNs6x7UT}+I0UEB^?yh~N+jiL^x%h4;{@ro{>- zSiMUhRZt_5o{FJ&4tIzegRVA~hBTH&%u0v8FnAGPNh@aGpj>E2>j`W$mLlgE3jZ=d zf;zC$T(!PrHh;hPea=J@#U@g>_7yLgphr0AosIIvn!9=~#>Fy+RE$;ipC9OAszk@D zOQ;Kqh!AVrZT!(k0$j$stmdP;7){1Z#J%d7#<$DpL(J`_yC`$LWMMn!t3(gJ5v`?gN$fII2MhZzE-~krGl{^g%t5Izye9WZN`MetfP(u^-^DNvgk5r8-i#6VfUG zSWC;T`s|;eAy7d}rf&P{;PD1Y#Kb(1A)JdHm-LUOxjzX%W_5Y-^6tvaO*3Kad${uQ z0jH`^9&%8&wO#vO6VCk8!Egs(%jD*vPzlfWd0!1lxwRHVV?yRs0E(oACdv5M60fw< zdWQoRF7;Kh=Z0cRNHn92K3kTggELQWcqCyow&Mmtkx+d}Bp4egE|!{a;JblUKU2Rz z_H;?SjGOhx`3E*y(@*!ztCQ?~gqy?fkd2s^^oeH5@-a7?>JqRK9nJ~x7oc~YN=`K0dHYy z4we_2;~gdW)`f={ewJi4VJ2#=LwnU>bv9m1#=|N5?b8zOTT!k=Sr=1f}?R*DY{fL4pyOr-BPo7U5RKMp`A22qMmcsfo0|#@j zdRKb_@v0i3IJC9iqzx^9iPqC;q7XL?cFC!Y^5)q-RJ&wTQvm<(gNr7gjt|-b_zqiq zvEI#@Z|uAuWMxIt z2mxdntgv?VLJy0`LkwC_H_q4cH8A6kX9fu5Gs|t$e3=~fHT9zUt)8k{IPfslG&fIu zFvhXvoBF@^iA(3bOlT8a*ker$Ca5ZiOg9B9Tv0#^4wsvpvXkH)3t=UHm;Lcr6j`K| zw-Fph&&;@3u|xVV?pcG5`d-rXHp%2|NQ>tED{#L(Ckw;Q%4Lyny+k~?rVCAvSl)I_ zjjNx0zeA82QA!|*emx7}Wj5^}Y1mT6U>7iQJ?&zLH&o9(lO+;`lRIu{O`B+$UDY!V zY%s~~h~;C`fePkGf@ziBhmgdbt^A_t~15CG12UCS1^4teZqq=<6)-8 z7hn880cY*~ULM<>&~2;6+>DviQB;#7$!UGOVH}@S*^k2b7Ch~?Qr&BN`EhtULZ%bg zGi+zkw4bWDS->4NexmGltWaaW=gSW!wIw0x6^6J0bVg3%j}*?RS7AxYm=@njfre}b z+0j4IY*0mcv0ff^$>7&S!3@4P#ZxLC&et&A#TcCusGz*`m$+4DsjuH)kal z9C5BcJt|2o`{RcSS{A2D{W}jNKm+gu%ia|TEH1}mUWEqD0n!1JF1|8&YIcT#dV zk$8zR;w&qvHHp?T@f&$_N_sMEz7b;=`kEZdQC&c(H+hd!qYPdotLY4$N&@%YnlyEI z@7p{KXK3h+&$=}Cj2%TXzEy$5=p60{fG>XbY5#z@V1%dXl6r+n>aY2YP99jlbKS-9Drhwypz%@f6J+X!_3P%A&yIFkAf8%b(52T2Cue{= zH-~Y1#;3ER+}BO9QNh7U_A##*Eu{5j7nBK88ymK9{Nvlv3{zGidD%%^}K@+8Am#V4>w5c-rpeUgvu6Ppe4 z%JG4NY7CX?e3kYu>L0w}IQ}%(IRjER>d7^Lx zV@m@*&QBg`rzU9~`lj3+*UsY_ zndY3{tU8?kaX?@Oo9&9##a2*>A)T%1<{jYAQeT(-ImUbKgMMu7ZS#XEOYVnJ#YO9<+0DiW_r3&`&f#}aPb=1Sy`nCA>v170)tqJED+m)BqBwq82a zSA-&?pENR1cwe1K8-+{@RQm^mpHe2}2QN~RQYn0o(353bnLwJ87pBB4-p;X`xSqWz z)zpmOo2G;P^Ck~#d3lw+y;jX~Q%%+yV@7)mGQwo%`>eS_87bL{2kMq6DdLJ9*S{QW zA>V4nw?E$CK{AGBY|3wn#hw)r&<{J3{QWQ3lo1ogZ_e*D<3iH}e+x7y-5;KEc)=?- z)F&Ux?zyHZ2E(L>!GBacr*!A2M|&obbAyRPqkcmww8JO=BB^Dn#ou%)iE~vuh&{%4jlOP(lNt417||WyG%SU z|M4Z`!bW1JtbO^ouMKWKBi8O40`ZNC%;=YozXXQM57^iHDJ!-e`x01PmVKy+hg1K7 zz>w7f#oEdc4DNYlx_j9ezP`$2)5iVx`aiB3DDL?m2nL&&*na);vJVCbOeVJO`m%q0 z8PCQ`CNLmFL*FkQU!Gf%%YtC=2@%oN0>v$c5bgfoGeEgR%zK*qmrXLr-DYk3fPgY2 z47W$c+iC5drf!ki7cY)mL0I1k2yD8Q9%`R&ev{qo0aUD#=JBvmCL~B&$n4;dH6o^z z`??a5bjSxEA-39~L#9z88l7{z{Ku8DN&8l;&D04o%A)2te!CRpN~+F#(>rsH=9w64 z_|l{9ByE+&R97Obc4ocZjgZkS;5FyN_-IPlvu0z*6+geCV%@rRW*G~i?i)62_>|Es zOW~=1e#vjeXz+G?NX*Q=tE6Bj={-NKz>1U&nVl?)%E?Y6o^m9k3Vh}+cF}itN4nTd z9>>i>7Urh-;G;;OB1e+G4+m_FdQ&F@fPZpwh<@|AK(32>$}YTUGz(qNdsNVVc6gD6x~%bb67x z(c{c*U4o(Y!*8C%sy>IF_v+(7qHVY)bO1b^VFaP-V75r-ud~3BSRil4u zQ|2rv6FtT4*0-;JX+YZzS}YI_5?&?C-x3Vs%W8VGiVkKzf)-zItBU#>P5P=VJKk3T zreJp9%!;-7{Jxem(4yOQySlG?>Ni!IvI8Xx_K{vikz5O)#YaaA`&LoTAX$L!RFQYB z(&9sNXtDE7`^7Kz>dT;E8mb^tK0~*;xk`&?Bmp`y+A@52+1LO68lJv+3#e5Im1Ct zf?l@0#PZR3GZmn*Y5-c2g-m4=e}|4@X9VctjF>MBC2S~ip|=&mvxUccs3ciHw%}^) zyvW#WY(_oQ7oP7n#5*~vs;v+%?LuDue#Byc>HE{dkY=sMcVXfU04XYkkID+s0TSBA zK!QD67~!`TW?6-_!7vsJ!KZ2Mf5^1a1pXwI3%%Ns)&6LqLt1gG8j5CKe%Edwc}G#k zp&_Xjsg7zOELy0_Gg|^`uK3(Ss6R4h%=*Jkb!~4vAq{0tYTUIMA?rQy2{cDwrcBO3 zMD&c*(N}Ya1@c`=yjtI7D;Y9JKJV+_)CbYOng~#4v^Z(qKP7i9k0Ct!@9+Mp@2kuU zK5>(jRq>w)VS@B%l_z53YTVaf!&>7xE6hx$2G7FGs3csqB`Z4rLbqX1YiZsaK=_IG zG^cZz4%>q*gLh1QN~>d0;9cy@NL&8%78$%n)sPue+AC0h=Lv*bF$76LK*Ul)GMt*K z4s5awuoX{F1D)T6Ei|~cvTX$K$(IkblQs}g2E-i=!9g>J@(N`jnOZ=n!})2j&e`2P9}sZS6lFO%Id11> zq*JcZV74_J3+^qS^kfsfg!)@P)b`IudIMCiOqa=rxm7@tYw$$a(kdEq6qb+h%RZA0 zPODK?A9}TC<#2QxIe4#OrFMMY{E|-q3Me6JRV@1ZTcM5v#ILhpe-JQErmC8& zJp-g7Pd(8)chy1U@LtPix40~y12Jv;I~#!pF*_nMQab(#fz)RSdC^ZkSE*-D>Z78T zgP!^r1U)!`+{z`mXOE5Rla}eeoVrJ0V?paS=hucXq9El4_%PWA`-Zu1`xiAJN8b1Q zhA2Ab2bRZ57>bx&9S<F}TYd)7{JBYU<(NND%{`8I+{Y=!czx;myEY=AD literal 0 HcmV?d00001 diff --git a/tutorials/llm/llama/domain-adaptive-pretraining/code/imgs/tokenization_diagram.png b/tutorials/llm/llama/domain-adaptive-pretraining/code/imgs/tokenization_diagram.png new file mode 100755 index 0000000000000000000000000000000000000000..983afbf1cc6b5d5ea6815c68754cf2193095f87a GIT binary patch literal 113644 zcmeEug;$i(+C8O!2nZ-4-6$m}-Jqm|bcd95cY}0ymvnb`mwuguuZe7@#}_KGAi3 z5dsGXPirC~B4;KdCSql7WvgJVr*9}^VD;5dUQC#Ui;EKujxJhXS65z~fo@O_TUU2* zl>W^#TPOLTpfGt|uddFn&M~qbY*5`thY)poU?ya7#?oYVt_z|gRf=47|8dPrxe`>>h#e*kCF|dNKnj$~K(Peva zMO>K@MP^I}LyQ_tL^68sbF?89SouWIys(XoY%eJ$Gp-o!tSulrFVCMQ*pm!JQl4H? zu-48~$ncAit7xz<<-1(@SG?yjm-a8io`-$L#}{13_;Sznf+!>GtMQAkDBdqpDZ4t? zI^ukMwlRf&<|g{+`CuR-J}Q1sN+Cz|^X5nFY3CZs`(!AmkDnYyfRAztk98_3x4duA z0gN1{p{j(Dv^3mX;5`Z)e4q&&67UWlc)bQ*zyN0<{PPn;NEYHh-y<0O{$nnW&Kew? zAe_WUAw_5SqjY3&OOV6L4~;vk z35U7*`Z^BtP`YR0kTxe2azT_A!s2e@kk1+GMz+SRs6L-#K&L%F)7>&*%eyzLsWxMF z>YC9Za^l`d|9O2)5KK=ZCO{TNpc4F#OBboI^<0Py9{b-fAADNDZN7f17lC5`@gqS5 z#nqeVQE;gLehG%*WSx<0as>5I{^uRx-$LA<{%7;V$@)>{Tp~{id)K4?^Nw)wPW*`f zrQI(!frbxUqi>x2mpgh7^JM;)=VJ%i01el_durqSU+yTloj(3wo{!qg2{fF=(~pz) zKgI|iJL_!B_`i;M8eBXSze9+Ky;;NQyjzR+r&skE?JK_Y z-9MMsG#RiA%zlkt$mA6k%C#?Eb3m9ZV}ZtrAh=a4A5bh`m~z@;B>xeFS*$?NR(wuj znVtjtjclD%@v(fM!0`W{XUrboH2x?!`)dqd3w!5nm=17U)w?MX7duD(Cv z>N{N)OLz06$(GD5v~=k__=vxe>WFI<5!HcEQoHL5q}{Tky}47) z@`e+$c)2$Bv)dU`wmBrM<#YGgIt3ZM`(A2XEe0w(U)7opiQmL@vknm4md3jX6QDbi z1U*2S!pfbK{v6&EU?b6%w$fDV;kRXIE2Zh$913qX6Fr>h8c&ZM7>tUTD-nhnBugH1 zwUI`372kERLb(^*oGmINzi|L#B}sE}pFI7g@3n8$+HGJ&-cWY!u)=B!Rb(vMtw5>d z77wdg&!%IUa=4qB)-k!fK(<8kc`eS{eZy}BR)axI4U(1*SVN`mfQ@xSCU7VjNud}2r=ID$-M60!^e;bLOt3MJw zS*a~$^5gv5N^U06mr(1mQ$$}b2!G=T&Q@;IWZ8)(D(d>`L-06;!EL`L_9mfZ|GGo* z31+PR0A>geF7HAORyk={8PKQFZQ~LzefRTj)q0TH9kJHWJw4pp9SLHG({+xsJC#HU zy)W%doFD33geRV!=L|qKGDlxK%1PbWn8i-!Mk>9gV|Xh^e+{S9-(}EFETSJa0pVwp zJt&24*k}oU)6>S^PKK>v%q?)T=X%j(R@{`?TR0>HSRK`QFvd0q^Hb%2XSevZdQB`J zdW9N1Wn5aoh)*=UP^R$5y|>EKljZ%aeg40)`UC$o~rfIMnISzxS;jkFsB-wh_j~Ew^G)G1ttMNY!^T}pR?}QHUT-(9++e7h4$?T&8cD%)GFNrrDEbj;BSUhL z^z!9S@)>66)`kUh$v~$=G3nFNSUE>rbC=m$!qpq2!y4J8p@T_2wb+~cmAH3=I!0a# zQ4d9oPu(r{K+@TD{kPRzn^KH(mmW6yLL~4E>Vj7WWl$w<%BmsRs&_~y(dfc#FBF@k9CpZ{g`L@ z-j39F&@@D(Q#|>ki)|J9wi2wLoo&){HOns6&}BE*)n~g9)p5I8fQHOVVWGGX$guDr z*|Y;@m@In@efP=kW{jhZl&-uX?Iy#*O~-mXzv4b!kfp=^L?^e6DU4HSHNXiEM9Jln z%qse7=Y8|CkzKb9!|G?P8?-V%l9M0vMbMK<*VKzIk?<(ES-*@CwYE;io`eW=W4G2# zRgs-{AJHGT{=#D@B}01h*hv7scBU~lOX}m>9g8?ngPD&~+G^rW-8}>hXS zcu-L?h)ns~=01>`&)Kum;1RlRTGu+i=%SQNW;{p}zNvcH z@V_iCFB?-f1BZ7RReoh&y!xRA&VKpThEkE@^1a6~#Ba3gJ!Oa-qVv-&qVei;U7^hB z$GQAQD;3MsD^I;=$Jg4JD5YRkeD6@sjcc3M4E4b5M*S+v-%^JAzY``noBBzvT;IT`g((((@pNB8BdW7T6RRe0|R2WZ=W}( z?5T!z-3jlqR*lO_+l-I$m_zY#-K~Q~E29v6{5#}c-F(ZetMwb{c|iYUr&a%F>Ouzp zg7Y*qg;%^&1kPqSs`ZXYsWdWq62r`J$;i*b>pUR+r$6ucp0W}O@<_23&vyf)+HM(Ic#G8tRo>^djjWwV{O^|CY)F!?0G+@SJf5EB|?%>@TG1zB*r~~ zkw}&HR$f`ain(23Oz9Jw%*sFx<8~mIZZyKA;wyW)-AdMusuT7lr;&N6!$fvT)44u! z+>sM5sEi!JVtdJ0w(be;PgE$lIhy$iMY%Ao=fhcO@4NH0@rX`3u(qtO+O%oq`ulDigP@4Szt zvAf0YlwhMRu=1-hk+v1uJF_YN=*xZhJ`%LgNobsoQUq4J9=REmsqfn>kH6_0n4WQ? z@De`zV4ux9uI~4Jr`=0rLOb>N&jeQdfAEhE0+Y{5R~qpScEW@oABSpomJ9o(00d} zqdMZz{Dmce(M3ca6d3TKbgsMf^eUTk$~nJZSY(+Z(BoHP^{vhE@Hs{v;pr7H{esF~ z&d8iY!wV+EZitCnGP{^Iui)0+V%XT64ob*nzzk@Yq{v2G8=G1j8)=!x+q74fJKh*y z-ub*;GyZnuDeJ^}9r4k55Z-&Yhf&5}hNJ`)ZQ$wBT1Fa*f-1qXPBePsM9X2W%Nfzf zr9?tJ+_0!tjBJVtelzRj$SW(l#nVdSeosNoG&-HBW+^mT5E$hRriGhaLj;m7V0 zS}R8v`pO-o@}-{mYNPEexNRG<+uP)+K}W!m4|6(t5!de_#y-!Uq^8;8`n)m8a*HBJ zdDt#f61p$oRNEsg;UxVPzoPu?ynz?zc}eG2ACN@;BdH>`U1om?ymSfz zfucZ{jeny}Vbr+ULSc;)yE5{3@>*%xblitj8|@88sYP|o&}3FGZIBMi*ERCWcOqD# z`F3#{B5Ba;Rm|bL4QCTo1uOhZojMd~<&|{}=Br-n0#fyy**X3MUi8at!f(M$2L6Gz z0gc}bZUR_AHJ3LDEPS8xd%=xT$?N?XpoRelE%LlF5MM(V$Ou%d5oGX!cvL zk%)=V{blbpl^G(#AYF%WO>oaB(XhehZCM#VPn;H&jNd1Gf+H-y@LNzNMc{BMT8H7Y zS6_po88AX%50t5lh_HYQlh=v7B4v%2pT7vR^3q!V9AhJbMT=vw9Md5On)P6ZJL1#s ze1BMDjj+|Rc7kSvo=0C2QID-D!R6sTqZbF%VD z*nVD--#ZNMa&EGiMmQ@5N^@_UQyH$Rwi zRyOwf?KvS2uIp715AX6FOFg4$-h=q&ar7mLuRls-7v&KGbk{Erx+FYgTtceM99}d_ zqxOC#=i?gIaY+Wh)ga4B-=dg>CtWmh1){S^=P;=&W|?0U9z-+h#$zQz)|>_DKC6RL zeVD1rQa-rm49)K7{WaMUTn1HWeS22kEmK3a|F+vKHe-iSV@EbN-VX9#G!KdhnT*%Z zK&TJ89k$Z|zfWX>KrU)~zT$zPo(ZNZD%SfI$@r4Ct_K?imH2_XZp|fRf5PAyxj^E} zzV(fCjiY`N-k}~+@_Q6Aq{q)Ye8p4tB4XUIpuwIU5!a}vol?@wl%M>2RgmS^I+pPl` zQr#Q<26@(aQSKJk)IxS#&UofpDoPKXmqZttAT^E~klYK9T<+ls{vX@+znC)XVN^0C zc<_OBNx{D{j&Yd)&vy2HlK>#}wyC_WgD?~BJ8U!hcK(gvi-5zK<-&HS{UfXV2bt!B zPYtZMu+^kul34%_nT}qaJK)=-nDA@8_qB@6Thf{NsF$q7SF7OaK}7}w_dcmMfO#>( z+;clk(yez(>5Vdd`mDYsX^lnBlrV`)e$w zMGA+zAo3^Zy^iF`Pw!LRzEmzsmwjx-fy|@X! zFmp=2Fy)g-FtdQpO5YayYN$98B~^s%vJf&JIOQb3x^kB45^l-rVs=bDE}!I|G>9LV zvdv?`o(mN*UHIIdAXhg??o5=`3tZ$?&6-z~)YYTaDQZe?YO5o9#?D+@uGD7EVp$n_ z{P1L_YWgBeSTdpxsuhB5OxnSlSZ`1rY=UVk> zOF$9EuO~?=29I-#6=yokQG{bFAA0CZt^MY-;6D) zPP}iJ7p|(8xgA4h%EgG?P!4`<^6Rl6`F@JPGtw~$sXC_f6iSDU1l~z6HsFI-EF9*# znj_w(*>UjY&0a=R_g~L;^K1>T%Pw%8uy_9GdjFm|&a$k3#2?Yi7!8FpXwsdO(Myi~ zt#OB2VAFH3v)0L??#1F=YF>`3%V&U9;O9{*9^|Oo1n5_~jzdFt2(oljpjamW9b#`% zGozCUAFz)Q3Gpy*6^eGBhoX7j;aa(Up}D$ma*fYDAY!WabZux796LVO*ALjuk7hF9 zgg^gGyqo(FS@@L{pD@Cs_6}aFRecS{Fur)A(q!^eqq#HAd}Zx{_w;7|?PaC)`8v-Z zdyU{FKq$Alt~N}uo2rUre~Z|257ojMmR#WuuJ(A;fa#M5O~@|6y6Avs?xLK+Zf*rW;=~wXUS7Q4xjhXb1xDY~Y;Fah zU($61vXix(rdIb6M((mO=EUceH!o61n?@FRaNo7F+&a|78IKImlf)sCR3Hw3PpUp} z$-eqb*J8(huh53WK9bDWP7Caew1iUv4NXKNqm%wYq>?;!L75Ym>~#jlg99U^lb;es z7^kxpMkb7?q1x`|k&u*)c53~6fR|_w$C|Zgc;3qYIJw$*RBp_Y za3`Zw>5Az2zO!HRiR`;jSV4&=BS)l%Ms);u+W!Z#pT$b+K_3U|Tkz_x`B|r!$Bl(} zrcpsRE0{KWbG2ah+A7Qik^U2_-Th{Mr9QlV2|wX36qX06Ny`frms_5LBp0KJPUR}) zXe%z?>)Mh285+|-*-b3G*pa=}=Axu4xaSq2Z)jp8qx3cSOHscA=iFyYax%7Gt zPHET%O)9}U@z-22eX!fvk3|$#S<02AT^BrikDM$V>W>2xI}^f~Ejrxx(_OPIgl__| zBfdKXeG2Rh;RWax_gz;y+R;TKt^_PiYwsVJbKicx0v|>Jbbf;T>iVsx>SLD&G1?1l zo1;oW-zVPpV(cSF;a&|iqlAvim_99RROG*)qFPkm1BkcYXYKqR#c$(ueuKLFR?6pa zOknN(l^x-ssgy$q^L_i9SXCsSGYlK1t@$g$O`ju1P6P0Us{Q`NEpJUwD$H|}#F0T( zgw9&_opKO#Rg0I-8Z&a2w5qA^4ZOIOmUVJPDBlZ)EV7q~o4#kO%00{p=3Wc9cqa@^ zEUKFuiOY9Ac^U7W&;>mAY8Xx~E=wbj?RTMWO{&x879!~KdhXD(roUqGvb-s!rTqkB9AU8hXWtdrBiN{X z^@_4(wS4x}ZF{h)?SQ;eE=DQ4Z$s3jw+gMXhjPkGHyX~vVEd@H{B@y!$M z#222=Eh#nJ5T!o{eoDIQUS4|$u_O~ELQrvYVm?hH1FvigoviW^m` zdRy1_c~#S%XLFU9xBbtQI4)K2Lr2)i${X22E+kwFw=V4OO$3)#PX2GqDwtc99Emkc z^ZH1>%x%RC^DKv^lMOA7*i|S%kIdM{t_u=Qm~_gfD&;yFwpzB8jb7-Z_YvEGr<4Us z6UJ@wTQ^~qAFsy#{y0?w?tQov= zZVV7v^L>ndGZHQ4;|5;>1$b+dpq>lt@O3P?1gaS`f_jNPyIGFQ$)LLzG{59}n05IE z?EK`ytZfUO;h(G$tHRm(zVolnF#K+!Sz2{>@~}7mHI7O7r**-3`roh>D0{j!Sy!E6 zIp_s;&NOSClTS*mj79IEE1L=cN6u0LHF0tud1YFKg8$Rx!;04rwdP8Fm_t)*`MeG@ z+?SPK2ixhEQ#v3lv2F?Ozx;&>SOL#60mQ(@bfYS636uFJ)To>|yz358+g%j?TTm@D9A>$fD^mPnTa9BtmiG_7D|1_@m7+pe5nz zr`afen2Wa(d-x(tWV3f|)X)^@CogGkWFHl81?AHPo`BI5E8W`^kcMxkUV1w@zy*JJ zv-Uca*6CQXZ5KI1q&I~__Q2o<#hT>cXUnA7ddB-OW+DN)nZZI(!!fn`LoFk!b zc%k^;2{fQ9m?8x~bS%>^QjUkBB9NNQo{B#^9l~J!g5xaIY4aQnM>W^`d&Co)3C^c;MseSZj6cZOW7^C8<@ zTYtiA8(3wht>0ACvob$_gNf&CL3Pl^TdsC{pHWviu7>34Z23#4c2m%8%MGOsZ#~0s zI#~=fapmRvWxvdg_UBw*kYKs@q*znnA1gg8A14c_!-CcoJ_uBF6BiGD*T#NUm3d&w zGWY6pn$fc_LRl*2JblD-0r4yJhT)hNZTBwYeU7bciZYuk2Y9kMbaT70e8~sR1S2_CU$laykHb#XcGCM zFF|5u>#C+eZJ}2D_^mC&*n!nRyo+f`Ki}o;W}aG4gkk+9dtqsnin73JmH*PyShz%M z#^Sk$Uv5q_Ctf3ch@>Ujv1{KX{-6|1+2D;*D`pxxOld)-ddSIj4>t6B0Sw2|g!Yx@ z$8>rLKQ2L2HOlJJ&SBMB=h?p8%R|M2IRV@UH8~fAk~Y~1u7GG|h_h8DVk|gu7|< z!>sR}nL}Rhtwk$V#2*c*>p9HX@TQ!y7lD+43cumz$<_W!Kc8?@HKi-&B0J#^i-rZ& ziF`m7!kw64d$Q*!VZkmw0^S}T`}qV5f;D`dx5sw)1{Bs~=C^sN?`P&Mwv_L8L=5d; z5n(9oq(rd87(F3%g`6zcSx(Ac5ijUMzD3?%y!>nb4Kn;KBD(=kpHzh(x^!FsDhbw_ zhM}4onvsPza3a%?adICX+cFy3x#3klEg%rt-3FvW?_TS-fGDq_lWAs(r~)*-1|s@cWIiK@(R4Ph_iin%b$b!m$0zG)+5$0p=~mCcf%5lf z%Vv2BBP_O6SMV-+SMihbuRdJ_PwL5GkjoM2WVp3iJ~fvT0e z7j8rYP)>Phi09ise7vG&T0n`BDtD^=_AMF!)DYRRxJ?W@a%`zPXYHi;u3||Dt$H&t zANVN>04TBJv%C}$LXD>s?s{Ri;h~{NKHhHM-4cM3`EJ6uMD5h8D)+{_KwHs7tD&`s zxwo+*^{P>h*8c8*?nn>vr9pXj5P+fLe!Xx4_E!^U$Z=f=I_x# z42wte?VQogg-k>F+wV0T_aav$>XbOJhtGgUMjoi44!>MLQ<6yi4KT^?73#p?sESDE z)t6AN)CZGU(0ey`+5XE`*+^LhDbth&jV0~8XQ$jsA7Hg@A8+g1RL2f(_|%&3>#&dO zk5cn+A7-mfAoev^mAL3<0=2C3 zv8 zCSxAcI%jW$1615g2KQV;#zZWw_BZ!+vhVCRI99(mUmBW6x`popz*urB;8+Q(V>Dg< zpe^g2EWg?*4w?i0#5V44ob+}?!S8tO6^qsRlq2TYA1Pum$lG#bVC>@?pFfTaKr$B%r~i*B+9nbp3lxq!0ss!H zO8n_4fzI%(M@^oDUwuqmQt2~gK>`6=hMu_3bT`(NATNWOpUQoX7K>ulrgLujjrB^2 z76NKDFhnJcju=^$`|i>!YZyknMKLea%7czlH%t;xY&qg9A5y$|Nf()oV2C$+w#}o{ zAmTZPLCmAsy{-o*A%|!G6Y&{Ku1adC!NG8*)YvHlL3s#E={zYDEL%&UmXwEl%WyTp zD-~G~`jUa_g6)tqrjF~BkU7o@CGKvU2NcCcm}x!g0m3S;v>YF(rm+ZFEh#9K=D{3c zp1YTTfbg2Ls;Q6o4hlkXE;k5=4r(#YyAYU8Dm`}(+-}_8JjE^m13(Uv@25|qcXU-;TykS^q4F9(GIQ$OvdyJWlQ?5gobnZOjkJt?h99}MxZj8EDh0O-UX?r{kkkhBS$T{8VC#@N zkgCWM1ryP%O;BhBv-~BI@*@C-)zdm^NGOe#nIi;hW!*)2uK~>W-pRAq!D>lhE~T%; zkWaC7^V1Q(L8AKg2)&#qxpo56mB1vQusd&WT^;5vcJL^`1sD`FwXQu#A3RpNYV#Ts zfwb)l7*#tB9}`$0;;H%fT26n{@IJ3avTBtT_4E6Kr#~Vvt#WZZ2DxhD#SPv?^x8yK z$3zMVnXgvXsGXbxrJFRM@Fhf0Kd}OyHZXVF2E89^*m4>(!|HYRJT~`@oYuw7-69k#Iesa77zh>K07F@4(HhtH)Z&GHXi_yZdSG`Ia1Vd5)^dc}MQds;yI2M?r0nIWTLJ z>Gf1Kn|{6^swnP##fLXbF87{>!6&P#)~PgZu!4T?N6CSDZ9e!rf<`k^e%zpy0O~fK zrID?$a!|bp&_l(NTZ*~TX`Zix(MVQbGc%tpgVtwBB`SaUAUO>_fHYR_LKW4-NP`xj z&u~?`S90=F#4Eg7DhP~uOOjLYUqH`79U|4W5d_%9+nx5;24=J#5knmY>JGk1w50a8 zTEstHZu^cbtX2&$+Nyn0pLKU z%jp#deTb!b-s-v37<~AB|HI89$YlBYQBjS=Tx|=>OGh=XK;ptPt9)~G`<$(#?xW*i zJ4S;ORq^@Z01ur(hFHHV-CFGJqv(?H;@79F+ljA@Sq^!LE2Q43$c})}0bDgoc)znv zdg(~IT%I$0ngj+_<+@r!1LO* z3{dYq(&x(srLSX+J3H(ELHe<#&eSKPu6x zKCpQqz<_j(x;*w}_PEU7NfpB#%xjwaW2>zaW};qlW9($RXz zJe!6qak!U2xgOj^h>pOj9gC-`3CsAeCg&x8%N)l37}DyX4GZKEr$AieK?R+(5{u0o z5~o}@q*?KKJl97#qwz>-_w-vw=2j@;{oKr{K0KMak3atxJS1yS^fXh#6LJKg1QDr9 zLNV~C*}*yl_2X3}Umu<^v>-w-FoBZsX%p}8&muzU?aQNrEm3*)HWkscCTrf-d*L_s zAv|MHOE(xjq#pVjtT|$@J!Zb^dEfYO5^;)$iv*NWcQ-8XnveOIFI4x8RXay=AfnA; ztN!>9hwiE>GdIEm_FLvABAm(2UCQO#9zIO|(Gq)=$hcr7R}by9KTgH01Jqv@?FW#q zHv~eh&G3;yawV`h%@3kH{yaC$e3^TM(kQ)xZsbs2NIX^RN0mu*ngpOcpU_WSG%EkMv$=3lAI9X z>id#~_hkz;Bcr47D?C%w%(P$RQ!dIEVz-a})M1EcCE)0GH>*FuS>Mhi3JMGRqVqE| zSKP|@QvMs#3BKD+8E^H&JsVgb7IP)qg#^>Y-m zowls3tR3mF_Y{A_VLBmz*+V6|6GWH^g^P1iUoDyA+x!b(if2*$o~ZAy$H{&0$ve%@ zS!^jepa0c$&OYE9nO{4PGLU4bf&LZ5Fv0%Rc#`?d1Av>)5&T3<7-5^HQ{HA)g1 z`B49@l4rCcM3zQ?M+kG-js3eCUFQc(!=g#jIn$2V?x$WOhjO_;RzfK%7pDExLFq5vIp;4Vh2KVGUS8V zY-z$+`>k~9u-j#eF%b(Y5^ARf3Bufs^)L^^xxm9|Bmp65ZvS66>c!rCRSxgnNzeB$ z(!Yrb8jg@2DtG||aaH8tE$&GD%Ac47tRF=qNsi~xp?}#)SP>NRKSwVwFRQa39`I?s zh0MSnY%c23yywoeaYLr36$!X(4b(=R{|>zuNi6yS1_lPxzR9y`Z&>xTMw=4TBp9jI zG`H25%5hiSW&U!YDE|)C5=#zRjj7+QGjWPL|7%IWdjZjY-(<~L}HMMEs z6#r{p3f}>d_eyjCM*`S{Pt)cZ48-c%DFNjVCY(r0%KgwiZVdKBk z^-gn`RYRcX{ac78p0$G=B#>Nm{8N*GaH?Pk$UvxhI7H6SA~Qcb@isht5mccd8*@M{ zl}dA;Cw3SNX!>>Y#^2umdi`7So=dE0Ij*I>UM1Nf>Tpn57&!R8W)+2J>jy>j`rLFp zV(|D-hcPcjsqM625tOKEv$G$(fV%N&RrA3 zM;RW`ohGzVas}-VE49^-eUh^ASU_WvcExhy`>$m%ffGb8AKdPewY@50_B#!W*Fwfi zgeulo7$R}k<5TZmxrnQ?A2?3F7jU)@{Hy~*UPwFblYsVNn6mrKHamacEw`wN9Y?hO z;MCz>1&*wyIY#tX23L%vpr_Q7d_iii=N9XSdJ3kn9m(bF^bP&*fp?ALd{i|+MM)__ z;&Jii@#!+59n$bIdj|lbZy>FNg^dHtWnrW676R{7v@_h&-!AK9Ov5=5sb<}~r-#5K zzLfIO0-BOH6{8($%3$~1T$*$n=eE||@s*U?H;b6Bps~I~vHBTsxT2?gp%9IKNrMEB zu;>!c+BZAhZ**@kXZZLzu8g{|`WSq89hg16_A!5swo&*kI~nA()?D-0GC<@$$?S3g zy9D4JxwD!XH`rX=e?kl)&f<8MU*SZUjvA*4{w&4xHG0i`{i-89B61xz^CsKu%ifaHf z0j5azFt{y5?z1Qg2Wq%p?SnG^E7ZswQ9hQnGA};F5b?DOIEs!%EAk)&MjR&qHg{mSM5ol6^8;3L$F>QioVCG;M3qhh>Y-%B!%pUtA_)g-xqis|SMt$| z;Zqn7WowBp4HibA6(f7WL*zl~4*p@>m#~hy#`RX#a%={7lcZgMgo_X9fc8V#G@6p5 zS3Gwk{!tjHgT+~alj?uA4j*-CvSN9gpnKk^wkBo2jV=1X&WzIdm{cd#1L_Y(YmD%b3dMudriUSm9>*R43o7sV0fS&hmC`T zg#`3}oxC!9K80~kH0Iow96#8#*2TfAg>lfF0nCqzG^RN3$1b-}iN+Ri;?FS9?xXeh zzi3q|oZ*`!&1>RuG(24rF^POh&zrl6{@WRM1|1juq-oS?s1y#A=%$F1_}$gMkr ztsd6KTD(NuwaKXH11UY<0~GtM+>bOXI@~4vaiG;AJPBVW?hk>(Qm+AmUS%>QbRVyIQ`PSVER(L%Kg%oGVP47G_zBzIm|1Ta@z>H$W%t2#$Dw;yjX&gGa>QSjJ<8L7XMor^OwXv z`Zp6w^n(ha&QxgjdozwkOx4x)tM}3e4Rue7!+H+QodB7c(eckw9*PzDvFK~=;K^nQ z9j@|VK`5~rbNFF_q?VE@8k>+RpkgZ~vSJP8j9a5pN&~zM6ZPp^b>19hG8_EPK1!|YbqR_Z23e=0tJZw zad4gGNm-D#b8y!YG0K%fNwg1BOQ^FgMNFcn`xYtd3DKBTli)ardQ^#c!WdcFv)8oi z&G}Kfg}U&>5c3wbiiw0+{A2#srg4HKT?8(B z)?WSoBx~mdK4@2HKf|^>VYykX{y39SCDK_vzOS6sX-Xkq^=tlIxVuEeH+EP-J3jMV zq-)-H-;3>Lf#F{=9L4gW{w%>a8ugoOH-Iha$^}qkJ;A+eY~+Nw^9pDOddJFB)X4HWl-P;6O`iE3-Z#1(K&G$Xq5|>M0m=~!e%zY+vrb_xVxkQiz5o1Q z0EfZgmu>@J5nj3s#vE77J1`O7c{Nv*9NRIg%I&S*YQmC=Bt6GYKPU6=i?3SE*l>SyYxz^ znUO-^&HC8_^XZENd}&flL?=PDCfuE4YI#ImE3&Ja*A~Y$g+I-l$2|G`weASO`U!*Rt2!)mwT zj|xXn5_L1R3%xd3lSvKbV^>+rS%1H!o@BHx>|UG9lYJF1o)UA&YO@xKxBS2Uz z_`;vD>arkbpc`LZGo}2oWs(Ew^vp@V%^;9_Rf)h!E+Htzkd>#=oN>Fi?=;v>bb?Cc zU8~F&d9-j#HjsxHF$fnbBm`~Bap+=o${%xJbm#SV8qLLQb%1)zF1{b00mWXHz{-MO z2gc)IqhqiHXi+Pj=#nbU-L#gD1y4gq8g=C2c2_LGC-Ywl#)5kdDl2>HR#7*`v)zRO zDQ>M(tQZYK(0mZNWJ0m-YU?E;7vb^fS;b<00LopZqytQoY{JPJ zw0+ESM+Jv`1Lwa0=L@z@gZ8gbMCmnb0jDW}vddWyoX=H|6yC9JHo#6#wdW5%9>gwv ziw7O%_$Pq*3$qIT#PY7@cH0C=g`34TzwfaioV9CIOr&bY~ntzcvx?1a4cm3@7HG3Ao!CB(y%n)7_Q(G+Lrm@A3@E zSHLx1ahx=Uwdmh3YUh(6OW6yN&m2#>!@=i)js=(RP*HpLjg748wB$1Gkt{C&J*Bbb z849N0kxT7dP%n|?gLNu$2XV?15@q||X4ufWzDYfy+BP{*Y^rf;|5sk8Xoaw zLv$~J84B=jW%uMr4pl%k?z!TP8O6g@*??Qmfb&;n%%(2uIen)aICnK2bzYD)S zKn^In48iAtA+J_ief7^S!q93ztZ=$`&KjOU2x7PBGd~e@nBUE2x0CI5Qu{raLy4&m zfuf>G;ExVb=N*9X*ULb(VrYMiT|I+aTS+ZLl(E^ilag+~P zEAhB3f%C$RoZZdd20}F)4?gPxZ7cs>V@Qk!jT;#@0&fB_;c=FZ}gt71kE3_r7-fdWJlLdq<(X>w2=o~xpgvmIW*+Ks?DyDx}p8te7a`NYwDTWLu+3mmvLFm85iE2!K-zpkA z*5PVfKUndI$Wg{3=5i9{2=!iSlT%QN@)E%%O>y5b<69PPxiR4%!^-57f1^z#_cRtC z<*tm)cUowBkjH<(UxE1 zbo_Xj3*iBE|5IGb!Vb;PcUb{=Chg@)2!SX))0T7`T3{dnv$9yI8djM=@pDePzJAoS z{JG3ErM|ac17Ez5H?!R$tnP!7@8Eg^g3E0^&{op7IJJf^=sPg6{nICiNsEkCJD&>` zCu~{kSe}9XMIce`HmOp5%o1Vw@v{rOM{q;VxsAs0YLVB6SBHlhsLMap#rQ|ErUQMXW~Uu@d8x%U$L4~o&B&`sZ8w~VrS3i&om?EFZw93h~8lOZe;b1(TXq?HeX zU!RO98+-_}@B2mtG%laTkGvAH8QHy~0N@1sk9axzxwSetX%Bl&3{T^f%DOwn8%>)N=3SWXU;fr%H>D z93&(31IVKSq@RXV>U5O}lNmUdP6aWs;mQpn2}#^pQEn_sRfpW|z2K7i_yI3HW|219 zk=FrMN_-za5WA7Gk<+Fil?K9FO;MFg+~d+t5DEa zH*BG*e4ktbyE;*M898y}?B=e0(Ww?_co*bKx_N4lX7Nt%&Vp7DydxF)!p-^|iNTUx zzsNElsXe-GUSNQ8b{)5>!FbYIgZ&op@c8WREk=Gr_K|e`R5XwbkKPOvEk&#;E95|A z9D~X6%gc?X>DEh4SOoxvt=tc1zFu1fGyYz79;oz*nTD4%sjh%uVfTd{gjf7XJwJJD z+Y3Ija)G%ZE-OW5h8q*~=R&W$ptu!nUIN@@WpVi^>X|Sx7=oeMM@d3n#_=r-+MGxD z;H{i+Ob?WEub9I4M@yl{)$)m3AcU1)NXw{${{*^)lYg5wa@_Ph5~vY3k@>|yr>WuP zMmS>Q@6;#QZ0MG7-6G+pQ@5;miO(&hn6A9&eB7_;<_oz)w&LeBvY%KE`K~Dr z2GynigvVFL`BH9QMEuonZlN2970So(FR-nE&aymn)( zA+?(gQT5IsZB4-AC{E3gn{2?B-D8suo|aqAL*CrdA@OWWv?w;i9d1yAwXTnb?Nz*F z;E-|U@!NVmG70br<&KP0U%oUIp*qZ{FX2Qn^9riQd{{jV7bl}}poqG}L)Con*l*U> z5F+$Naaa(aMYn0pd}Bj02FN6br9?%gG}jMwiM^uvmVS+ zzp#|xh;*qe_XNcd(apjCIV;Vv$yW{0^wf_drhNZKnzLl+>~?2LpD@^wP-x+0-o19E z9iB-$$*NQ5wXt^f{dn@VOb?y-Ex zr8aNUK&v&kP(F|_+M#>)omdFD9u_iYw?CzldoaU2*Z_*q;LH*0X3nI{V9!t(6Yhh^ z@G-xyfTaeg%&B7R%tu%fACTqo>v|M;w>+DP4S~sDKsiOxb+L60T#< zhUa~2ybf)jyTa<0Ul*-bS!!#2c$PjALO)N05^k)@HaPVQJ4te9SYkC%Sd&i7Aob!7*&gIyaZ}A4Uyz5tdBnCxu;&Ib1|J62As@`iPEC z`bS_2DA|yep;HxF=V8%m9T_sH%REwYXsEdnLY6|U={x41YTDzXtmwW~e1t^gSwDco z+XJVpPn=Mm_Du0Q$;FY11m!>X1sb~+IM)sIo6Nv!8~9Q(@PTN=2n&}!=7+adlpT8p z%k^yz#X;k*!kPyJUqZKk(Au%mG+cQgr&5>%lNnkaurTUsy=!Z-TQT`|*yHoE`1_*L zHCg5i&=)4Nv&kMofzLeV{(&;npR$vyY6pHRg|um2&kW*z?X;9WSO~x1upbC?3mMPU zpb0V`U3YqcEncI%TiX-bs6)o4)z8;_d-IM0SL~M}wG}wOWB$-8s6(~wE%Nn{V;a+^ zPxb=J8~e|Q*@5Q1UPcJGrZ#=Fe6?}_{z%K{8W z8{;{~cRXZeltFQF5daR2xr=SF`z4n^2E{C_o`OWFt38B(UGKB=RscbvYRW3-OLMa% z`hAB=$)Y;|*U~x=T-PQ8=)~rQW@s226c!m9Ib!uCOshiFl4o_fuG*irR0pDiYsoiU zjM_3|8ss88g+4K^wvClcNrLKTx9#Xm|FHMst@HHX;+kLxP4U=G{B@?c>gwDC?^a4q$c;Nc^A}mYhC6 zs(<3&6(IjZtvCx4%;kua4Y?6+vO`>l(*2vudHh3(04e^i@a~churW#`eE(qZuXRx^1@k~1d;Qm*tAX3gNp&yfI z(>_{37$f1<1^^dR7Y)3cFQvLzEI;C62J1F~WDEWvs?y?Yta#nw-fg6r8}0R^&b~H9 zGEY$?I_=EYyH6-QbU!fg4T$rtp77-8VeL)V+P2`PG#G3s>aVl&^=~vH1{Ybt4 zsZZNy$#>qXu(lF?p*A8xg2?Ais-$)VSDzmjKHmcXebwM|pw-1{8WzU5Tsw61G*Oz6X{U6! zp^uGx#%bb`v$08332jPB`wp#C`K!h%lEeNw>Cz;hPGJJN|z}8e6b96{a2#N zIDNcBRHLO!<~n=FmUF-6HWq5%*qxjJam`{a10yvlE}Y3kW`4WvioGN44~usycNV3= zUm?Qe2QQ29h%F;`E?@~BZ(8-6b^Cd`UElB6>!hBHf<&#Of=!>|dQbk8*tTa>BCYL= z#Q&zp+Wz)fAmDwd!3g1sgz`ZZsES$Wgm^!vj#TOLWmxW!S34x-<0V;#U@X`0^40ry zLmf4E<0mf$P#zO42|A1@UMk<7(VBk^$b~S~(a43xtq`G#DV6NEFG*Zt$F0B`N8(8P+LCcGpcqR#upS>*GQc^^p2Fo z>LPe;T89__|Bx)KjpU>izsT&K%-(!uvJJ2R6p$9+$t1`do_U2i6aNeq?4JZm{!RNJ zO2dd6PFlOrJwc&IYD8xW;+Zp$EIInZu!*^kq?;)xAIW0e3MqOXI}QPZlFyt&Fy_Fa1xHq?FRiEMc$ut43dkM&Kuo ziJ8_mtsi|^2)055fV3&}uTU4w*2*>KYJ={=F5@Jo9gP#`rGt19uwC z7vxa_A#AAO)%`BdGJ-LP$8=`o@L>dE`EB4G%dM=<<76Y_^I;rt_;qdE0m!OC=+ zmuC0qVNb&f>*k7Z?o0lyt5-;x5@Q!%cF=NJVm6+I2(U6uM!Sv64)|u>_|^WB1jl~9 zf_3^wFkpNgMV+h&#zn;Wx|$mhs2_j1;AYmeg^Z+m&#q_qyuLNmDYVzi&&_>0z|)PA zYtM3}3vt80)fxG4afRci8_Lw!b2}T%_MQZkQbywckiNQX(uRHJpMlomNIrtxTw5Of zyD{tl{y$J}QVNb$eTOKKGhY4+n?IW-`fUb(QD3F#K01XT&f?I?Gvoy&(UULq_q{ra9qagyIyC9O1iYa5JYM6XFoFObYz(VyM*FYX^880GA51QWf+ zyyZSmR?O0ATYEvW=SozdNXTHnWTZ8J^6U)`ai?hmrQa0>QNi{n?TxROSG&K?j)>6G zD8Qx-HwQ;=(w-lGQF$X@Z01IML>2!6Qu*1`fu|&U#Joy|x&+m{yY8gnp5ZiHqw2o0 z;pYlpeE8^(wXIZ&n~Qy#b59WCDs^~Q@kd-jQP?>sGLRRt)~}#&0As(n|5`Fg)!fkN zZ-J1&1*E4;jm2?a-KM@}%0g`>5z@KOU@G#ac32|LHb*MY*n;oF5i&fJnl-%wwVG&r zxa7Qe@uO&EBx)gxjbB86zR6GKC}XdwAAo5s#1q$}RgGd!IW+OR29+45n?PT=gBc5YaOlve0*Hs8p1 zyE-i;m-Z+DevihN9gaHnci0T5d)@wSi>lx;U)$e8G7(r|OUHzwOs^1~6y&sV5cSK6 z63*LsAqnuYOOG(G@JmdiROp*VI`fx(D;5>uaSBt$V8-kmJhloxk=Pm~aonDacRT|K zHQDNm`YlI9eSMTD%(`*j%#2lHeZSwZG)3TMi1cWn2FOwu4dpy9JnImwQ0EM4U7t!Gu zrq`^*!k*wd(#Fgy(<5lrHL2X0OeJ$*4LQp z`#7Lya(hhbVcJ$}wwAbKd$J3s{()e|+O(g&8TX;~bSF`gLJ{}dYEytWQhw~`PG8!U z9eVSOQmG!nDF~+~>JwWskooI!k^~eZj>}x2WkVc3?jtqN$(cQ+gnN zE6xjAUae2&PnGl|W!seoKMgG(p`$$nQNUaG4)Cd}xk zr)1I8DCvJyCqvZes{MRQzIFGCd!n~4Li@vrmD_ zGqr7Y{DXJ`bs5sC1Ilva2GLd8K3dlz#wx0ht7Od)w3 zrW+Jffo&$&hs^C5CwLEdLSW= zo7VZ5CU+aTqMhRwx>}fUGa^k%6?uP%-5i%HP~UMGU{a?+vn(b{ZemQYHf^$7>=KTY{Up1m%E2-ct}jR%^?H ziJp#n(%sN;_JrvG=ofv@gfu%jpE4N6Q0!i@_56|V(iBo@7VW9cA!lQ}C8#{_tXB#f z!f%l|D+fD8gASUA<*y!exfL?02+AGc+WbGK4=O=Hn}WY*E>-G4n*)i2*r7uI7V!X; zV#g2aO3ph2-ybUjg%i+)d*#3tabKxQ^TBMMcwK!%HTft|Nd{utF``qaPc}T`vB`lt zA15jjb;o5W1yKYvvf|hTNx$4;ccevE0taoKtj?VeE_Xpy_JZx0uAT!O;`B(r^)4`w z(WAL5ZE$@Nn~jy*-;%<+;n;)6L$&eQo`pn)mp0?_WtN`6Iv(EKg{wCBLa5nTyX_TNDGH3I z2}JAl?$)v+&9Y{tw2E95U5SiR&(J)9?R^i}O=)>8G3u6@hwl~r!vdq>^GgvA$#s6A z%dcZl8E%DTPS8RoAnqqPw0a51w?dc?3FmXi`ysIZ>TVePzJ+-}{fkQH0LJQcOY>56 zXfW^m4)h^GFOSK;H=J0y9w4&j8@bo>y;{O9&<>2sd_Te^(L(oFwlOHE#Acm8T_nlF zCS(~s+FpUN5$tY`g2QtBWf+Ok7l2NiV7<^cG^6V2rj%X3Y}(kL;;!=EkZhf!e@5Zy zS0MMp?^n3A_p006+L^UT@}7vGJg;1UET1fKm+bf?>Vu*ZQ*Vu~A1bxZp;g|S0CiO^ zfNRO`cC{Pn>)-Ncl&_s}3Kg@i%EPmcmsN;Gk1~z3T4!T(F1jx^q=ugfgnd>rQ7)+v z&t}q4u>*FPnaab8ma5sfb@4+>~C{Q&`+^9R6)*V=tgjNE1X0iy?JWZNCF zI&VMyLkN4v1-3k0$BX*e1ZRF#FQh>#k%#mzws|($zEa$me%V>enAkgNo+*xgDv>u2 zIjU5Q*lxdiD-0iD+I&x-r9txon$bw{)w0%u3-6YD{(If%J(=zau-AX&dIFuto5aO! zVLv+1C#59$*K6)7pNI10M13&o76KEAU0+TkNa$?=lt<%Tes7&R0Gx{&Yo}E-a=u>4 zZmou2#k#J+nsAE59zeE)-^!q67qq@T?;b1d8-9keRsFnnRuF<>bxZE$1P(v%<@X-? zDjZ7&)@*OOTNm+VOZqS~5}ylAF5Cu;c@;^=Y!u11VF@g|i!ZG>B(9z^-Q4tLanr$# z1EkPRk#(!dw}HV1WGA#@8U+wu8BB<$e&8%?ks~UaD;Z%>pSkYX5{d;9#B+>Eb}xFL z+#jt&q&(74+?ZAr5&VSb`a%tmLis3Z&_Z)sG8!||TiJBAq9Ja*MKx=Y*J&U(Z-8>PUcBz-D~b*L)!UJUnoBet zd4JgQvn0!{w?EZBnc)NQo`=c$=1!Rxxj;oU(|3U0m$O!d(%wIrcb#s`BJM>2P9@lB zM>qCY$yNi00%oqCgyl<1kVr|9LNa;2KsaLuF6!-V=0G>OmbYxxWxJVnxtCeh<3ispvtFriRLp2ZPg7sQhRp@&hZX`I!Z+~UOA@Ai1Wv(-v z`5NCy53$tuNy_OusWJ>2RC)P+6=eS_Ll>6A%y;eQTjwJCn#!oeF#f9A-A#s=D10p- z%u(geOtm+&vE$HtdE0l6N6Fo_5QXQm;p%i7mG3)S)QhiAyR%v!H17451f#V1sV+?& zSM$3Kvh=Xxp+O--TdiZo+-Bw9OMDkFPW|sQZW~Ue$bt_kWCSu_W5C;$NO=d0iz)T5 z>VK~9{;IqLG#FE*i#7na9{6_Tw$HyB>zQ@QRs-;SXUHsKZwRPj;=^UvF^mh%qM-SRgG(d>W-ma78b`{d=sxxq@zIN_bm^2HOSv3E^0h}Sj|D| zg7Lp!JlVpFb@`M>Rn}m3oNgxEz~G{ovbp_yebTt;5(_+-zOX~4e%bS)5}U%~HREII z%kt>d!C5k#~QLWUkB5OciXjaBeIL#B`;r_#kGKoFQM(i)`CYx=7TV2p8ML-0ej? z<@r|Gv0Sc%O>ZuMn|XNsXPX94dkH{EY7ARV`e(-D4MXA*_CQkXgI{NGDbVwumg*8i zI3Q%!qvy>lNLlcl;PJ%^pDbMxZH2h@F)K?po-tmaQ&xL@*cQjvY|)Ba<*0i8G@jev zX80SKq8y3E!|HnEp-)$CXa*(Om@2o3Hkq0CWw#mm(=@2_?>fM0o-XdgfB%M|y25p# z!cZ7M>8hY-Yr!Q^y8>C7B6aOy`Ji>v(dAq3>Mhg+U-mlyK76fn$_3;Il!9x9RT$NI z_*n>w$FTslWH0zxUttS1tigCc1e5DCFV-#kS|O2!P+8@!2!Ik7L6`3$h)M;+Y-sgg z*rQV}@K5=Vq6c6Feh8L4?FLu^^~Jv23wiB4B^ICJSQSHR68qic1`iL9A>PL1df=UY zKU+34vg`#xEywK)`btYG{#3*>VcE+*GwBZEwa(a4k!%wy-%Gu7Y~$*>J)G*bf*QvI zt-^|W11G)By=}Pq*09l-h2b89#%w(Dj4+#+s9CYx4YOw1WABrt+U~*Yg*g!lBlz&o z7%XrSq)_y)C|d$!0^v*Er&@JYhj~nXn$pDt{H|*a3Vvy&D{2&)VpkQehVXH933NKL zrY$^LrJ}eNxwrEP6T7+Q!LyR0xpqB`{XajH(7+(_G*K~OQHNNcIC|cuvk48W|lcuOX@Vd_nb+8Cs)tu zF*&n$JVZPvHr@5uUV(}p*chRxQSzI;HJ zI_Rz0O;1Jk32t|OlNB{xs-T!?KnD($KDESr;42=NDg z$@vAx|731~)jQn}yZ_PgxmGvd7etahZ1kSr+AgC^6IzNiTkU>lc82G`EXZ_?P{Itm^EmXv1U$2Z)f~# z7q7+iC7?-DL@n-_$T}}VbXwx~v)-eZcbcV37F;sk@*1t6DN6q|BNP=d=$0b9BkRk? z+r17?h?u>3fvq?beoZ;N!AmzK_R3`d>id(Er8-X9Cg`;cZGp3eavLQKo!i}^g~6>_ z(O2k79~w}GqWBR@XU1drhX_-in0;QpnhVk)lf(6!{`!?AH?y{SW61#PD_>@ZsrkRw^Z0J_XvB?r)p=)@fzfp@k&W z7vpRx?tWKq7X!%3N6sjP+jgL(Dt+5+&lwuymCERdB!6R`MdrE_(+`ee9X)uQZjQxf zFAGAJ2+;VA`$TCsg6R^0f<4+zZ<4Z0QJu!-$sgoW4n1*Utq<+RvcWz{-o+wI;_};CoteE^*WFbYkP?BlEefOL1p8gKt z4Xa9UBk~&YhRfsdAJ71x)1ghjbID8$Ive@@z4I05X)&4X;9VyD+iwR`EKmpqkyT6YiA}P?yDp0!XV5Xf6 z4~8}1f~q>GnBV?xMvz%+v-F3-Sge){@w)>xUvKfoyA*KYQP73;%!pXrZmG6~0q_k(>vBHE`6N6W0k8~ADMn}^V4x1%Lmpng*oA6((w zhXWJii16TU+KP9)ap?8`lIP&e)Zb+22)!|F0sg(rJ0E`2bmbcVdF8=Flszu*QzX%V zC){l$5jR+qdGO2q1r6#<@ZCum?CJ$pa|wHk?=48;M&R7VVfH88QSch~9tkMTGpW#~ zT$vN1MQz9vI8|uX)~Z!>>v%fEcAnm&_j`n*Z9Jw$u0>NIly|B{HO^+epvwM1!IlrC zOi~Nhl1dnX&=@es_rKTfYA2wt)V#||lYf@CRE$iMi`4Y|iSI(UdVRsV!U}LLo0wu> z|5Y@dY|HqWWyAQ^t2Wt?aYwE1j*dNiR+qp0RC=_Flo#pjxsCRS{QV7oew4>W{$y~> zW?eJv6!vX~8OMG)qd+8O6RQ>4UJkwC(X-7Qv)*&exFj7J{tv{+-E zpgPa2F_JVKL9E1_XK2Asn!VFh64u$*@BI`v`Js{ifrWtWO!+Rd&cV1g!^yKUZ%M!> zY1v;t~ix}fT1|A*O=eR!iXl%qdB+-3LW??x60Uy>-m3!(QI;^+*NiOb@WpQ+`1b@X_ z9k!R#545{)}b|{6g_s^^Q$TWH3+^z`Cqs%f*&?_sUb`qZrKiKj}V| zB-L#g+R3p;gx)B%i5blvY{Hvn_OR_&LRnI@?{r(r$vzKMgkT(FX*Jj;g5+v~B(-_R z>s(;w(B`-IQEj%O{`Ixjvvd&=DR4rniFSlLMQ1&AEU98VxQ*SLI3fz4N+A2GvfN0b zJAu;Cy#f$0FPBu9)lw?vf@q@c?Lxvkz5L~;DN#Gh{wTPjIDF#nswz%(VoN6wG>A@2 zzroGI&l>%CZKyO${j~OlqmTWz(|giQ1(9EgJQ~=R?O5a?LL|>^`AeA^3D6l&(v*g= zn1k86toE5sp9Uab>!ANiYsHwWGA-YBJ85+Ily?WjWPnaR{mW8N81*;Y;s5oYbw6QJ zo^7v@`<#~vkuSN_*GrTawcp*`rPFZi;`e)Te{WvbwKRlEB<MJ$9&{$l>nRpeT!>|P(+jfJl7g7dygo=@`)(}F)TyMni+S|(F&*cIO^*&}I58t$WZj8VJd}7`jS-P-4mZjBD*X zD+2ppMA?eCDFdfW2Qh$4)06I{CUS-wyBFMYnJbkK7cN5FeBg@Fvbqz8!M^JVeRoL- z2%>EQ`>66d1*^t3jmN0XNk!SLq}s4jQ}CNl%uhFKG!z*8Ox0LAhH9^DL0WI3626JvN$`4^~XbIgij zf)*s+86N5@!{WJ4H%$<`-9Memz#*85j8<49=P2Xqbb+R~Gm+gmfq(;-^^nY3GeKSbJrrN}uT>+#txMOVKGrW?z$E-MN~w4n7;as(NU4SmXQY zcd_-k5*ot#VN}qp&(%kzT5J5j7ZU?fk(7Qb6#)Z_0-XUarhhN+(r^$25sjJMd5KPo zDXwo@FX=(^iiZli`|>5ll&ZABHZxWCj#B0)v!|LBu`1RGOG2@WF|-lfPLwJtvW#s{ zE!xlyvBYC!2x+(D--{_PZppML&oipg`i3m8NAb{!0_V zKXT%}fjIV%`d}xH@S=D17~6o{S=z+^-!ecdL8Y7aTnE+ldNvwS&im zq_ItND(g7v%5sDhjeY=p_)CR&$gG!cyy-tmFr?mkuT;aOMfN>*J2T zP!x6c-ZDYoCCHDWqnsB|nf)IE@9NfX0nmdn;J`(OU7h(=62Fe$m%t#3D57OHc0rHt z=K@{)8Z#L$_??kwn0s80mNuDOHq-U{J%~RUU}vgJOau?aRcV%TmaQ;qXoqT2Ne zs7GqKjh|(CB~9wElc@uUWX#ZBFpk8;bp#$V2Jv;!&6NKZBxEmLiH{VgU-<2z>JG3e z^Fa-<{tZe+*Ve-H8$mN1}VyVu|lnh_8uERls=W^rK9B10BxR z`r_qW+^M8Ya)j=A^MMM3WxuWS!kDzIU>pz^z1t-OIMVoXQ^$j1;zt&{Y4Qe>Ac?Tj zkjEqEU*;{^2l4$*!$G%hrWqEl7f^?RR~JEfgqQ(Nx~sKOsrtffke^jm>TFsyRGf;z z?JxW+scn%2=KcN)3tt)a#w$A~%BPH{m6~Xu%$d>?$tsDb4)tN2By*#eog99bE}q)O z*WmjXw#WYrhJ^$)k+WY9qe8GM(30QyRbl3*y2pMh^3~N*oVYyH+uoN*7*?36b5%K_ zRw_4RPkMh4@=4^Q-nGq+V?1Y=Pd$?v4W6Hk#Aldo!~A7K`cN|tLFE7DveT3Df-Gk2 zdGU&KUnrAIhxYWlTC9&-Y%=K!r!GlP7!2!`Q^pRoc}gxXg3Z?y+$>y8N=QdXr*I+5 z+oo@=6PJu?fh3NX7yBhD{`GLuVKVm_(k81zX9>BN?Us9l}*IYYv7&Ms& zg&QvFni4@t7O_e~4hOp6d;Rqn5>l)*VZ3yPS;m7>O3c>~@h0=NdW;HaYb4MNK0@Si zK~5`_un<^bIyFE{W2+bWDt59%3 zmL9xfHwV%|QK^f=tku`Ry`@5T@9Jw35#@@Th`9bzHop|{)_CX@OfY+xlf=>r=_txK z5Hn)Y*q~0crZk1o+2W3VC(Y$MgZ(rm)W0|Ddv*~Oj(tWcW4&2TliXUs38vKSE(lU@ zw!pd_@29+tK9HkL!ww-J0*EmNzcp&rf7&HSElf%Dz^zD&_zP3{OTp1C;KyH|5nDJ{ zuKn!*gb6Xe@q1JN3@T{jU`Nxs+tY!KkDHuo|8BolL^Hvss>|%@<@#-Z%UFf1%w~?w z1G?mv>wSIdbT-5#+w8r~l#a~CXY+4_w8;kdvvH?_dGV+9EC%oKSP|wQhjxWX^Z@58vNYJ$}A_ShZqg>mB{KgBKdM0*4j_lJ`v>o`wJuZNP6JG zqUb>l;Kz#qU^ZK65k-F_EiLV|UtB9=Vq$Xiy>V#re?c4YB|J(%Rxh3cTx);ZI$j`n zBZag)G&Abf$d%M>;};mVfc9v#QT~|z8$VKDm@l+89e>}5EljutbZ0i6BRyWM%e6aO zZu&vL`-bBm^ZB@DkyOAak#|h1 z&}soSHQwLst5khd1|(Nz)Rw{_k{t1e#44gc(?PTm7nscd(#W^)u7~rv-?!W#%WrE} zKn?M9s?@b|OWT%zFZ5qeFGmJU=BH7$!e2wzCAGGFLD*VyTS-V@(Cn|CF<_{+o*8ap zvDyAxq<;mQUon9Bn8J_Ov;Mu$&EO1uE)wLDIpvuek3}tKN>Yvg{^aT3Liqah@%vB4 z>i>2^er9kKl9BP5V!^-06|}kk^Mj9yF^kT!CBR8ojQjXg;Sw4auS=WjgUXRf`DTle zw7Rt5r~mV0kDoIleLE$V{V|>Nmxz*p(jwR5#_R25l?pTIy2F}<9AQ7Wy7&P3g+G+! zyBMIpgKJhzf0+j-R6im<*ItIw#@ZF;;Qw&}ki#wyCk9V*WfJ;1oNGh=7fC*kwf zP&*{=lP=s-fS;(BDoKyLShqHE!DF`d&(&W7ev!7+LPn7O*DsPW-|)May(AITixj$d z=Q$f>uiT}z)%s&mdhvjbKZ3WCJ=Fd<+2~yO4};&x41MpaCkr(d0p`6~is6E>RqrG9$3U@tpYVa`$MeSdq+KAN0r^`Evpr9ejomNGlj%r;z@ zlk#y3d#s1hnC*_|{$M;f=^;s)D%7;-h?$Re!t~3@p&Vi)L4@CoamlGOE`ADlTNZ&gGv#N2QPTNiS?b&twhhU z3ws$em^MTR^P^|{6(8}TIRB^;c+pgsczhIHq4j%aV!f%XWs4))Aq)1kQ$iy2dtWVQ&FRxWqCHEe%s-7 zK`QDyZ5p1W*5+q?TM%)Q-ju19{_Eo9*ov;b&BeYC>w~~bGnBU6*W1ihh#%GM5Ep2b zn@}4J)pP!5Xg^~}MF1LFfFZMSBH(qoIe%rkXo-Tu_)Ws3fNQ0}H`tr~zNNylCu@>G z7lq=O*{`@EQFi4-vv2G0MrJo_Dnb??>3nxW5{F5T_ThTOH^*_K+me6b#~)&8MeO_m z48ckrnoD0Rlg96{nHfwRTko>p{km=Sd(^jeV%4m=yvik5O`oEweFE`eOI)MJp)2ko z*TY7XP(RplW3y^PAudi)kY|3PDD@9h94<_aS(Z2VrY%=)CJ@(v7BDDb%26j%LqVuO zh}Lqig1ev<>Akg0$7X-Hs^x0wv&9BS5Qoznp0FYtW|g#A@jcJJexoNU>HZETd$v8a zH7?66Z%E^s_uKe1o{STK1bH5sUJU1s2p(}l znEBC<6*F}lYN)l+$tO$IGb;d~6W1>HFwqH0r?Tau9E!7T&}e{UswC<$6Ejyl+7ciP zdT0Oq`<8a}yZ_@rdIcY!NpK}Ff-<1Bq)9jM+g3K&Br3Ch)lUQbF1}J&t@#Y{T_a6| z=`ZDyFNH$O!k~v8X_1qt>UahXO{;iYl7pI`4F&^yNdvQikop?DE`GnTt zCIVI4IBiskVV^eb7{k${QvM?XI=zf=Z~hybJH19FDmwL?AF7>bG9G#H_HsJEi|Py2 zJl^WCC{Np@k}bHEj@48V2tYWHz>2g--ph4vmGSg?>Y=Y0X$CkW!}3RQ6=>fA@LVFl z$7$cG{D%2M+tR#64hfK|ptr6aitSElkS=y2%JoSlFz*W*tMJ_a)R#=t5)a@gp*T`% zRIVG5f%Ecx?NgLwhGf;#n#^7)HnW0?u|WC#xVV+n_pEC3scFlKfrQ<&fVxWba*GF7 z?K+Q&1)^y3=l%VS9AB22DZe*)W&k-Tvhy&6Cp|LtT~S-f)Zj#fCcDUfzO5JXk(B6& z@RRBwzzH}33n+7Ua}i}@=JkiA2SPbsz->g`Xc|5HD)%IT3)Z*Q<3rVz5_{A~;%peJ z!BG;w9vHiCgpi^lWP8oUW^*|)UU08JoZA#;vs^w&_psAD)$_mzOTjc?@nsbt62AxV zf3-E}%|?^euIW~msTZH-75q|@#fmH5yA6~c~2VZuSV5CQp*v~YuoNG6`wW?+s-OIDKcbfwSA@z ze4FfB?lh|=Hr zn&ajQK{1c#dB2IHM3;pfsfk~>ykV}!csCsm{jZCu^zns?`ni{%vTNSbdO;!R zi;KGT&k+<|Aq#srgi^1*NiLKuaq8Y{a|jOT1kWL-OaT94d}3uGn3-l)AaD$iDrVDRs;q3}Y_5bb1cxg~$ATQg~eH)pE$f1=Kq8>DH*Zm0$9TA4mV(m%^940x}XOXkCuh0nj-9IgbZ=h=@qI$rNs!w|wlg!p` zFck0UZA`K6M>#B?}ekO;0WtB;GAu=1Yz^o#oe>u1!P2wop`)Zy728F}`_}cCz^8_TJt&%uVT2^Rgq%+U! zbf-~MPwpOT&vm3(NN>FkNKLFRKFr9p1*{JTHYdsk)#@f(K6>8M7Srw~(AjWq#fJsC z(!y2PZM9e`v`v-mMfHp6h{{ zI=p}4Viq_ae_^@w9A(b4tpS>epYkZ}WY&&R!~*{56O|%xxqzGCJt$BVxQMT^nv!uJ zu^g{=v3?AOo#v~YI=!cDVKm%mCgTmY*IN2rA(n=F>$5SpOo{yFNK&z?k!N?-ebCxs zji1GOOO6?z^1Uj%JL7EL`Yuo2@J)V6YPExO;!7(YWWqW8i$VUpmkmL$L*T zQ-%HAxn>{8a?@6d^Kgh~%Kmh5SbZ@^IFTQ3NNQ?zQ%wC=U!Efsm-La35cxvA)!4zU zI3UBdW3q1V)VKEY{S+0`-JzakeHIfQvA=9X(ps}q0Zx6X%$zFo+~@>99NGtTQX|;%w6elxv8PbdPYJ2J@jzFP0P2z^)NQszFGU1S^+sWz1paX z^gGfR)h5r29F1a~(?RwTDHWAin?WhMuihzJ93?d*Q zaX=-95JbAAyGvl`1}W(-$&r$jM!HKtLZlfwh8}uozMJ!X@B5zf{h@2Qbg47*%=27( z@87=e8TAB0jxw%%kK)vkU7gDY^AjIYNWRLq@XGH;-osyq$p-=iBF_<=25j3F zy7bjfNzW)ZXELpVY;@Nx8V4Pd=A#cZ8dLXyDM3cN!5g$zF&yh{zopdVV{OeysXFIM@(!Q(*o#dey zm1j2V4nw}+^+g53;g3wgGW^ZrxMz<<0@5>|Km@}YgC3<4$2&j&l;R4rC z9w+CHAi{!{rfYEMsmH0Ea-1CZ0;aNW*+tff_w^;Zj{B1^JSr-+b$avB_#tpmw6zbw zAGP7RzBq6s^a<3AvnS#`+8meb4y>ZWN)inaygCsPxI9!+UZa;uUgmJb#Azml?J#fM1t%))bU6H^C@yK*{ z*~sR@VX1PX9wKdq8u05%f~ktqp==4_?LjXiRt$q-e)UbQHPfOK9gy%bf)GGbyWe?x zX3=1T&x}~>_vJSKxC80}hEhsIJMJN`#iytEb3C<~QL$-Qm!DoPs2l{X>db)y#(H$h zTw_d}=$k01oxHB5^euYW)N4XQf6p;yX`R-&S_V>ZvQD{qmTf0~s^=s}GWv*7p58tU z-Ka8C;wy`7HZLW(hEC3+h>oKBXP4Iq*QTpWKEJVziyR(!2Cs&an7Q} zNFFsXmQ)Kl#}5CM+VgRn!pKOURiq4ZjDzAq`LwkA>M0DA2{qCMY&hESv@WrAC zppfE`6x>#)XyM&a-!Ah%z@yjo*bC5TEc-~KcA>7O_wg`LkKa_^Cs4=VnUn-ba9a8_ zhZv2+LEcSfat@;)2vRJRlx4)?`X^1~&uN=x7G^!YPO%L72zbEfh4#R{@71#f$glHU zR#UHC1bXY~cQ=Qq)yX@ae!}vn)kmy#c)5Q% z5$a1dZUZk2eWW(!Tr+HA^o>3g>bPL~6V2;gV!H;NgLpMK|D25^!XSN$O#mrM7eS^! zy+ZFW-0x@o+oAlAaqQMi(&ucl3_FP1TQ^0aQMeb~~f4O|`SX>|* z_q~5LGf`(5Fcw8HV*`H$Z%O5`ACU^`Y_}|FR6&U)c`MiZ{&Q*vUJFdZxzJAef)(p!ZuK&z3f)nGV>TiMThs&XW4%J#kpXTZVrD-*Az~x2b%Z9 z&G%FNQutMDXBC$DoVM~<_ZZWhJ4saYj;Q*ec;~?3SULP7H_m*T^*(&q#pCRJ(EBP@ zkxPtXx^A=+m#^Bn4am;HM%76tm52fL z2x+2Bpx#|^@S2o+{8EOG)D%g*`m-!r^^OgFgTqhe$nJ>$%dyPM+>tUD@sZiv*V88<-a;vvY2_1kY?nZsO_Xa@#{3Y ztIK6AMHJHp!=Fo!lu9Vla7o({J2r(GTOL1$bm{|O^$|jyAX}a2y6aXhmTG{hW=CL=C zzSjNRoV@)%dw4OtC6uj+!JnXcw+(-f^H7U#_KW5$e@rw-$5I_R6T?YH2%psjJ2C^A zeaM4&qiBJm#CPT|nl@KPG+w!MKZh{sXbGIZ%B+d~4q@|YbVnCB!aXolwH=k$R<3vjjW|mu03m{=>k6bas~8R2BKS))s(Yb(lgXwS&4d?uwXcS0BFIZeZ~LVKsf`O!Z^C zYOj8nYV276fnitJ?_~#_VMfGCB+u%XM+2oIUbnkHG=C7h;yj>MuXjus`3?*Dx<$$; zuM$F5kVn2KMV7x{Q9$LOa>}#FR`}UhBjX)0%j6Gg^Yk$&8ZC4Gcv#dDa1njRH*y^b zWVbez1c&>Pkh8U+xz;HKzwv~iTu*uU(ua)nnq?*RDr1Y%!V6b?Np(|oHkH!UA_NDW zn%s=#UuK`B_SOEXb!$@}Ac2;-c#`vOvX67tM)IHksC{yJEOYi_nqOE|Er-2PEiVpoWBkB4qv2HlIkE8D`xKbG-oh^4z7ne-6QC7qcTZR>iS=Ey7V zZxoiuUSeAQ6E328NQSwn<4Ui|HZ%aqML2r2#qBEV4{H_P-Ng!a&r!aMAt z_bTLvO7%rfPnIGb%bsanJI}i2K15yr7NZK_D>~0KC*CnKQ0qyTYjo+dh;h0N>?Z}f zDz~I1qhh6^ZWa$YzSj6l|FdeUv4wQcY2%@=bEDs9f~hMP+&qP+2a4Hx>Q=8Qm9LQUvL}e|dINHY*91 zOdvk2mQWUup8ZMTUm_)9TD$TCss^1oyANpK!&ib%d@HGBZ5~nIWeK}BX1*?x+a=o@ z%E5X$?LZETEWh@Z);_NCI`*2!7^+>kK0K)=F1E$9p9wlJd~n3%@VgU_l=~r+q$zDk zX~68b@&ogvG+IXeI6f7z_BLO=*w5?u`b*OVA9>^a{dK2bg^Hsrw_M6C`d+UKo|>$5 zyt&vPt?4s98e6~8{*i{~qg<%cqiNHaBf)3Wc%&lJlGMxSH1V&mg)B}sOO|G;G}HI? zi+MkS<+5(bAL;Fky?r@>(`>!%^f#6X(r^K70Q}ew@WjM9+$w^*$EhQ=<&b2~93K7>?0KfTjvZm~?$sZkXtXIDx{D73%6hnJFRLIG zKC?)tj7a==A{CkSs5;*V3++0Q$}-TIR=cQ^I7=4D=w5xVkS%fW6wPfRW%;1H+HOxz*1+Iw-q89)@!GSz$9&l+<|4mlA`92-){2U)QHo=Qk!hNi;En>WFE;Oc3 z$jwS^?Mq!zNz)IMPF6j7XO4P_3>~eMY7n{jV7f=Dc?jK`qamK_O~!53FxdHPP;IaA z1}bOVfs8QB)zU|+r=#LyQZvxy4E1Fu|C)#=fm$sPq^>-Nw0Z$uF}pHop?DfnitoPCx4 z_TFOL`+U24sy~WaWaQ;ngU%4&r~IBk*|1HREOmt^i-%k>+KBD*oV`~}F61nwT7UkR z+m+@1bQpHp%Tp6ab;0VD&E$^$&5s-VHX~S<^}*vC#RdVre3r-NvOa7%@NugY7C!Vx z*yx>@>rLRas%*rein{Oi*XZ6IbW=NvKv9cayGnDSw3sY^^dHtG+Ozfd)#Gfr?C@J zi|rDPGx@f1;ES%GnAV7f)LMGQTUbAcUKN9g3!@YK{8*Le=z>WiMKkH8NuAMYo0_U!P!UxXGh;V+oI?Kb4~PV`g3;}JW0 z!;d98joB8X1qx}V;s0_w7XyXL@5b~jiLzgVi78QDbY@ zhD)EfA2MpTgQ(RYhFEg5F5V=x3Bn>6U6KNNEw|QZ>Z>){8Z(xdcwzHG+o+sGoqZ?l zIU4QFcvT(3ILlde@m>6o*GEX(J~S;myTa09a3uX*uf>jFi*PuQ_G31t5Qdew=KA9) z%bV;VKAJ{sqei3BjQYls92m8HWj;*14X^QJKL3RwVO~6|w#Dy>Y4ZK7hp`*Ulf&&= zmM7D18L?)zSxhp3*Pw|e?%tM|p1P*Lu>o18wxoWlzOHGUBCFo1lIwF17LKpvVMT*{ z;E!Fw50?2Z--l8=xCcUbWZdN&Z~N^iXHG(coVBL{1|n-tMeO9cxKL`DsM7-ysKd`{ z(O*|Q-3^(x`!3Iasq>46)cv|T-G5?}nYW5`zIMHn@XSGz=XrrK&L(#PHS@^0u6N^Wvmv2Z6@rIDA{<^9F3NiK zfE-b)j8>B4SDtfaxV$(RG~S(W6}~|4F0}huC3D#nv!gm<=bYy}HA;ry4_Sw+9zV_5 zkna_dYIDWi4RS~0!yB`mQtiQfYw77S<&ra}mZztGTM7zZ!o7!)k`mC8@NxRkyDpDY zT^56rJqT<*!R)Q3D)XH~8gs#(*vb9Xo?O;RVp8wrqHaoUgPIN+w?eIr$GH=v!nZ_}L9s`p&^s%Z{53;yGPBs+ezN&o5xPB9}Z={BA+=iDlCW z-GE42zYXB#pTJA*`wHG}{SJGYTl3fLm^J+)Do!N1m9Q{W_`dvnSrMBWm+*dl@<9NR zB(I#$sGUVD{na+pN>jTwTQ!PCt{d)sm5+}6Bg@B9y;OSokAUr2H_=a`6-qa*8HZSn ze%s9sNq@cCV~g}Gvn=MU$A4Z_%0jI(I+CkDPAbCk>A%IEt!DUggmdhuB9!V>)=sYm z!5u7(()p@byM7}BSVDh;FCgkVtNoC#__lM#n>FJW67Jok(I!7@J8k;=Ii3DiN0%7i zc-&dJ;R3FM`9-OcSxSMWtItJG!iI8Fw%PXr5QOwX*jA)o4?hskj2}_iD4uF*|I(w9K(m@3ltI2lT;5K+^XZ{KYd9pqM45F& z`luf4K93y!+2h@aXySL=_;A0|d?YXWaP2qE|Nowg5$~RNm~6bWMW((ojbK_yf6Cjt z1+M1Yp1G!zMSSJuo^M`%3s=gQP&|RMCAD%!9|u$;S~5d8U?e15cnjBdr$OL+>s2k# zFpV>@=LnrYH}&DVHchd&+(+u3T~ zc&-{)$5nfB^7G&Tjtn2Y9(_C zvGucJxFa7m_1jsG1DUF(tQKOVY!MWXcW&4@8--c$ms$nU5R)V5igrjPtENJQKx{s}SO;{w)QLvIA&Z>844>pr4N98$WqTdz^ zLsk%kde_u~&#gjO?G`Dg)|UFaza*YwY-A9m9o4)jl!~}7h>%Gs|EN}{@C8i;eZ*SU zJF0nY^s82*9ZGFOtxD#ZRS(;*HOiyn-nMpB&V}&eRp*9X?L1P^NpIjO`d*!n0)>yb zL~JU&K($84WRJ7)(}I(%3xifU3D@flRNsTmNVh$;3kg!BSG+IXirHbIR9<&qtnRnD zB~tpGJ%RVBx@)fBV}O4A)p#v^ESZtzMXXr-^6H?H&o34BM{Ge%xFmybTkPF!=H2MS zSoeGAmZ2zk5dGAD^OBqK$)|=cs+%r|Ps_chBY$hQQ8V5FF?HGI$3Osw2Au_D+c5mX z!iPwPRoUl;ayi*9;p9Y5ta=npQAqXIQLbxTZGN8ay7Oso=>gSuJ?awW!UR^0LZ~hqw4BzyH-nXh+_{)?~#>Eiyg!2CR6s)}5)zJj3v-EG8zP(1QN} zb~#!yG^!px7|%1{D=7q6Wlj)h4SwQ-JV0A+F zZpqV{)9b%}_Onf#8$kIKPgbHDz@2;8NNDJ%&0w;EG+kNVVqxMmU0 zeVCPJBNw3+7sEBpO4we22}<^ll-^rFELV8w)t>lvxZYSgdy@0HaIcu=t23#WuvEKV z#y#HqPyZE~3-RJm?Bu3LJW%AtLd!y4Gj1j*w8o?bK&}GsbLH&Cy58i+fmLXMuSMg) zodYnHlmBA+u8;mmh-SrY%IJBfT&t@WAWvn+jBtTV`*@=>B zC4C?HGIaw-OJA(^UnhOM!XMQl*}+NXJr7o=io%q*@C3*Z;Jf&*yc0 zzf#D3QM=waMh(_*>8&6#?{szICcv#>N?rYqx*;mTfPXj~=I3^x);#=toqw?YZrOpV zu4mQ9iPF|-l2Ja1YaQ=PNAg!DVQv`j9An;ugb3xHIZJ@LlJR~yo6+9cZ-unjnWx$g zA8jU4l50nBJNp8W{#g2wuN^5MNg#cz{)}1v>nD|zU1Zxj-ozXeLLP%^xMu}`Je z&R9ZP`hzO$hwFU^R1IK@xs#)C*H!X5L?W zzGiAKKKnzoMe#jtl6=i8BS!7)&pKgGkX5~oF$Fo1qi5S&%yIIhh+a1K(y3u{NR(oRQrd3&H-tVulJWQ1bIe@jdl=GED>p^=sqi#AT*Og&* zdS5B3`GH}Hd}i5IN>uU+ZyisyqrG_X@mI0|nqn)%d{B_Br&`93=;SmI z(E|x;L5&if-+pvl+W&{qq+t$!bR`JHmGS_|S^&o#`c#-NQ!m(Y1|}#@-71UtRN$@b zuVqCR_=cWy^{%1kyxELGa*ug>ZIA}9F zz8^Y&rb5LbkL@)6|IM5xvWGU1o{59S3MeyR_b))P-0a@+V1=;%X@r>nipafU%=j_u~ z_KEjrxKT?>egpy!wN`P-Er!0nJ>#o*^^XKw4|6eHD47MdU;;Jz`Oq;%ECa~TGNvSu zzYuVWx^`}O&Nz;*3p~l{J#sC}^>6giQ+e4z;|RQXk91~p&3 zYl1uYeVnf9W$Yvi9nT$4@4WrUdFqnmQYWeWP)Y*UdrjUCx3@arHNJjxYJngraABL| z=5%S|`UYqn`}Et*KeOrJJuT+WeRbRR+4(QS6XzEZfdg*B0IdO#)<8_87@WqCtZ8W@ z6Hdc$zF9pudMqftK?iIk+wC9gQBFIP41zMX+E=OPA*?vB3mkrMeQ+OqhOr}oxK`^h zqW=AF>)M$(D0qc(^GEY%zFyWb7dd%r;NWn5w~vTVO-3>dtIhGhwS7Pmw*%|hS$mrE z)VK=Rer``Z4feV^DKl(HZ<+Kb=Ztw4EiGVpn z{(bgR^zr$ip{xsDY2I6g+tcaqc&X?p1Lid6KMBIsfBn@%aXZC{Hjqxw>OID>(f{Lh z4h=y*l276Z8mKF96 z@Ks6fw}-suAq{>5n6okN1fqY-=zUEPIWUx*Pe zMrWiODdQk3!i@!A+ft1=w&4ocrsFVYdmR22+n9>1nx(bGEt)=-+x6A?$gSca2YST| zMGL32ue0i3`meqD?aVyCfQjV>*i3*6@(;*dNz`WdcPkEePS7x1E_4XKSRAgew}(yA zcX_UN*}6$yc?7hjF?Hh6T~oC^fhTWG)vF^DH2C<^Ief72N%K^RPPuWYfv90TD9`%G zeDIS6bNGA1Qrk9eF#^Q+H4fWT%1Z?bPi%S6)zqt$_)ryV1QLL7PH)%Ql)i;oXMH)F zcJ~b2dCFy-`T{2Gvs%Y|YlwdG%hNX&*|r%BD7E6~p@T-ROBBOKO|oIxw@Pi0`n?&L zo$H!j)qj%chCoYvB`dy$ooDh#)jneUl`8cn=Zm-NRW~_%PNg~Ew?b1s1kOX;x+$G0R)5Ex zs#d@JS3AbY^!B<_3cX+b+`e>7RsBw#loM*!Q*Dx)c_jVhRaw@@n3e}*F5c@j-tNrD z=~3?nV#Rls)KVpQ1Bql)d4!3eY$*-SUTf|E0~?9i*{1hG=$<{muc2o^9|i{0yX-C2 zi7!wZQnw0v*$AS3r*~&!<5T)mm*TW!1|c8Fd0g|M8i+5pQ1F0QFyZzjuxP%PZsGe( zFPzQpYE_Op&%nyu;_LcJqxr);wd-?ro>y>JK5?N=2V+--Dm}ecn1{(SjgepOkBhC* zJK97IzEiL)sfS`)Jl1Ep1HMhJ-9-Jgf_6LjVh)v_&gEpcc1~oN;4iH#bqlQu63vlX zM}fVcw-08X3t93M58P+nGAQ|FYiI=k;Hv#Woj3A$w2%IT9qUFk9tJg4oJT+hCI$KM z(F0(j1*b1CH$@-*`PaR!1;+(**vb{nyW5=g`aE9ex(qgby1&9FZA&oz;`@22nrn!A zC31K+lnT3>ZwUo-voW{EeY|b@mUT2{pSvs}2UtSC#aH^0+bE{)eyUTV zNve(FiWW=m50`c`a&{!o-c-)vKxQA<4C8lM=JQhET>QCezR|1j9C*d5s%qk;%4E5H zEz=Z8Q{(v0tLt6PmrHR`Pg0x5YL5(SbF&fVGH@cT8Gztq8ZL3OAMhPpNByX}lH(n# zAMI?7Oo+DE9r1T!7Ji)vOa3@pJ7QeLCS%VPnRW&t5Da<55Nj3K3<^@$Ql*o4hx+j=rd3d+@Z2F=XX`TnCFYh#5Vxz8v#!jZCfIX53a zoX?1rzOF{tg}yiseVaJdsoBoUE$X0IRL5s=S^Cf?b5k|@6~`A49M&$|ogb~Ba^;03 zcwjKy0?9Y(Ta3EhE)%N_Y3^JWsvVtVuu#l6fBc`VdT6WzGC>2~T|J7ATGQrvaSU3F znh}SaCCy{7MEgO0bg2jRq+RLy;}5_NUk2#jZ!L;-Zf!YgF5FWgHAK*Rh&`@2Bf$n4 z$H~&JNk^1ttk9g${At#km-6TkaaPlt%#^F z4J=#-f5ufz6~rjKjN%%PD|+dVc_M<}V9&A1uVUX@w3m}RHxew&;;Fw}Gda zD*?&kR#w?(`(iH05B|=ytJ|zg&;`2-aKSX|Tiv9RVEge3(v#3AoG!%|-W zs3Qu@KZ*jjxNj;HYp9&oheaF7aU2&NyU0!7QMBDKf}pq*=7M;jpOTo!C`9)&u)-o(zSP(>+gJR(=O1rA0yiLMv`(O)VTeE5CN$HNl^QXA1e zk944~g>$08(5_>6DKUW^3Q99lE4}ep0lcqGdjO~S+9ww&+zbS0Biv@%DLe21TJLO6 z$DcK0sQGA2llK8pN;)`LyzR02JJ*wiz^V=gY}+GKEn1>HmB+5nIO1RZ!vvup3bXAItO038 z`ptl?R5Bwx9qOTnGlPzp^737FZsFIrB~GsZOy{YVeRUB;NMkT$f5J*cOzg<+hdq}{ z3nKqv-c=rRSKX4N)Hi?wGzkFx(25T4{IR ztq~h1c|6P52YpA&fCmK*^1X2$)HQ#UtRLm1xW6v;Wf;2wJ+UD!Lf>T53TEVw{6tl| zI#1NKs9Dpn?mjvIT{--PTa_XzB|0oOK|@qSyj?z>RKmz*MjtWK z>(2pbAcw=+(7Y)p17r6a=t4$_nozUe>2A?$=q=!-*AUrV*XSzLA zRSrt5=3_xT0f7{a& z2)@=NTR3p`e7(b>-X9#Vg_!c?x2;17B4xw5J@$<}zYEjYy;Pg0-+ z0JWRcS55MD2jH(CL46X%ib>dSQ|i^k6tXpC_tAndAKOusu=oqqa4At@_6h3T0)2#7 z&@h$li%=Y>08(`-p;_AKxuA?^&Yn}yS~lFhnp>I%=*wV$i2 zUR(xDufC<+e3Fh0Gr;iKn*lz<*|R|7yVMy(`1jOS%cc(WA`|b7y`)8?PDGN-u`M)F zUGKR}e-m&+b5cnlKcl6f2~nY3vwP6li&MOMMQ8pTu|QOg)}Gs>HItf;=|C!@|F9nU zK>i|KL3?jMf5Q3WF`L1&0o$EH=9XjB-1T3bE`VLHtMSp~SD(ItD$#OnIcj;L>z#bK zT#iM@8FX*>zcC;$DED*_rnb(e804MsfDfcWgo^{}^549>L{rZ}Z8Hb0w^JX6Ay*+T zOzMs}RA)9JjCbev6$SR%cDY{bZE8kp!!{=GDf8TQUo3kg_SVpa$dTsl|69HP+b|cB z_K;ABbDh}JeB96>)nOY$i-Scm?#j$a>q4VIXmt}5&@^E|;#}qzs0oZG;SYb#P1w-w z9Ut6U#l_>s$MFzohZFyh%Wq$D#B-(*h#n~fl-JkD~XA<~9p>4hH z6aM$%t`niOHI{BMV?4I$=t~#Rj!nLtevQ!@;fQ>%{GutSsESS64O!9WrE7KMlBW5- z>k=JxzuX)3mgj0qr1_kDoeeVBzv;#GcgNA-NW1^+I7V}`*!AbxvX+1ntV?kqe@ZhQ;| zlnaHZT@n(7kG=%Y$Bokqz35M!n2QMTxVJmHwFHU8=R2P?(VqvM>|q#vL#BMCLh)(S ziTm=U6~5C3=Y&{{;#o^yNV5pDERd)#uLksO>5am8{t$RC?Y6jV&i&Tr1fCTErcNO@K zreQ-Ct6hg$Gk>*f$hyzvyjXXQ8QFAziyaQ5*kPqP$pIDaAj8#i(S zYqXlSOG^ilza!Sm5L0d3ae`u0&4r1{M=s1T ztlQI7n{vOvIAT;48X-7ln7;exRrwqvzjg|gbhG$-^JAiC?{EtF@SdP+Q(1qpJESAz zTr(Q%bXUDTyPyT-Tyrh%hBVKOrtMM-hn9Y#L0tFD4hX}zT`%^!161>1*T?>y@;W#f z>ddTBC`?svOgacw3i$j9;|J4A{|d>OB_@56#}di)*qaNwfTCvOSafGUqM%u4!Y5*W zaKity;b3d#A6$~x?H;+eSNsYeFER<{nA@2S<5F=v&^C2@$R%yS06$pNbn&XVkS z)Z|=*L`nktB3_4XQ~J|C@Q`PL05sE{bqeh zfwN72_{xfdgMI+qb>BbR#C^yq^4^h@k-}d0M z8bTqP-+0zCHrhGP0sPjPzK8i~6V4umh0`~J` zlw#j&2!7RHFkb7ZbW7gE&NiRWO!|)7`_J0_h;X;%Q-j|g#_5!#U8il9%FnahK8Y;r zf71}ihMLRKVJRcktT}pwo6#FeE^K(Cd*>&bAPRbZ>qoh$a$cfV?h-$iBwl^x;)l& zWXX}1mki-U!qVa(`=!Y)Dp8e8JBL&~8;dMBLoXU*Af0KJ#tyRUlqG4VAoQ8Si5m2M zY;*NjOezN#^gsOFh{#78Y`fslfy{!zC=1t%V>mMH@89N5#uWd+X*xh5Q~chAcpP)+Kf3_r zp!kLR%yFNap0kt^jU9qa!N{d|L}jq(o%M%j5{xzyul;h9@#btzaBCHQVi^8&dicVK zid)3}o|K}~5F8x*)_Gq(6rU7F5`W<~Jc%?ouIRmalB!7rJIhk(5Ag08v0&`0e{G0C z``%mLV;y0nH^pHq@Ad|iNgY3^O-QFbk&t10nM^KRJ_Ia1QrPaQx&9oIV)oErCNg4rQ1y5(iMMru zRz)SNGK?L;{e0UnV;RVb2#1Mda0`Py!j(?rRX#i3E=0d%lRzp^$V#)yOb}1bxDtkR zJbyXuB2i^Cr}NV9;YKC!CEAO{R8By@2y3%mDSxRApKZimLaW!{Ry+?Z3puOvL>^dZTOZx?(84^e&*E{NHxSXWb9M zTeoBUf!^mS>0=1gfH6+}V1xP^Oo+Z5k&CjYHI-_#E>Vlrwk@A9c%g&7%cBn?R=!+% z<`{=v9VkwJD}oTz-7wKcY`2*?6iJB#2s@+w6M5A0KE2I}#wC7X7RHBHM9)w+Q}3*r z6CYz1=hog&K^NCdX0#wCApo2IG&{~X{$9{U^*#l+JYeH9a}eB-T2y{_?+=q9VXWgO zga_YJtcShi7r`r*`i_j?kTKxfIbwbc^K2f%W$e107NdrS@b~u%%+Tnf_gCyhnz+Q*p#iwGT5!Yeh*94C zk-IM#=PJ$rG~}=G5fnm@T(d4BXi>vAyhX$}oAHFCaibKjMpbo3&!6=s3rd(uv&G5q z(&%_hlw=11ZOZ}%1b&r#;NY2SJ|i?3X6yCCi7x+`vRdHOpSds}k7MB~jj{l6s=b`N~+ zy+n1y7?P}JU}2GjhR~}mY69&`pw5sVUBp_05u@;!>|GHdpz{bNG4|$>wW<5CpZG6d zt2i`^J|8}v<$m6nYhsRD^(4n+0sMU0{gZQy!auN&t?5LVg_H59r=1)$ZohmEF@58U zO&6<}BDHpUJP9b338EH*Ixwf#?)ScQ{LS*C(R6S&sByJvkpKP)$z8R0e!2uvWoTHb zWNbqux7E%#Y*8T6?iatH#sUhnXER6_=>}r8Ln6t}Ic!oM^({JU{wHtCOrzrh$Vda# z^}5pF!(KIAx?tOon1lVSD>9#tzw9!uZAr^tRK4NeE{9>O8$iSK2L&@ls{KUOcF->Q zfU}owB;Gv(yvBbY7N6ik(!T5ptO~-ed94;Z>go;n3uiw?NP|6H^4d3i<0+{v*$M0^ zC+#^lu z2x;D9rp6SGI}k56iVlmDqg1N%!m*i3Spb@i{^)f5|xzW3ozu(TQsS{Q%(fbtpk`v?khm_(k# zwCfYhR7d)GqWk8WHcerawVTEDw|6k)sxJ*4*h?|cFftec+^xwzG;AXGKoO@VP-mM4 zL4f|6AC`u1`pHW%84SdRjVOS0;?@*RVUYQe-zb6)I5pYP7>NYW<5irvD{yxC27z7e z2Z8M@COIa3oGYn#lxrS_s4!q0=+{9Nns3Lt*J?r^q<77|K%o?2reE$4;L}@2Q_qts zwNpwu!pCzpzud3s-1#uAdFMZFHaT@mMW|Fvay07}g84<1ezhy2x$LzbPb&+8vAC7) zf#gpx_Q(Vg8VQmIm(|HLVUvj!JsD9FL$F2GJvDG`A)zAcs}!$9p0kkDq`D`N7KylE zxg?O5q|I6Q)2#s1Koy^0Z}sES)LPY1B$oqK@R^SL_8Ujka@{X@Seu${=JuaC!Py6D z^@!(w6!yQ^pU6HmXT3W32$o0rOb|Y+#wPCqR%|x=IPKQ2??mA_(}SPS^_Z~;*xF); znNmg8qUrrD7Wm_n-_?6*`uA53LW+OEH(U|UM?>&^_D*Pp zfm8q<$?K=Q1RsET2+h>gqH5=F_??KpW#1v8uq8ftO8dUeN-!Cl^Rp1g3$6KA90No7 z+0}B~`%&1YwBPM<7)wnaf|K7*S#naqqjM ze+kewlBO?Lv|mfxz+s1voZVzMwb~-Pj=^rMC?bqUPEd0-Xg#o(?TZ#=x9#lWjp%KQ zS2XNiS9C2SPTxcbRGdZr&?Q=k7Hfr1(@xog?UMh&Cc%czj(4sEq9U1GWP^|@I=(}X z^S{^Z{Vc|WsXX`No$R_kpS%7n^CLmKJAKSnVby9Nm4{HS{Z6ptb9F}Gu>9%xl;kMG zF&@h!P(iC<;%`LheX*w>3pW{?DAN7U<@NrVJMqs)D``_zBj(N$zed9Oz9;yFX{fl= z$XdP@(&+y3yCh!C8)1EuLIknh{xJ4f(5oTg-R4NXlE;04t}MDge6>Q-ne=+UGEI$H z{)<@nfG&E-{uSsHn_Jl>^6MDdGQveN>&)n~5=at#bfoY^-T4jpwcfj`gSBmyx$i!T zrRnyPQtase}W<14!e1D0C?SqQ5SxhrnT+<$t zs%Q~;&P9i;dGKa}a6b>EGd#YkWD)}HxYnx+#g_ml{d)l!b7fR$z&=bo`}nE}OcXoq zlJFR`n&TAh@T_+2EuIc= z5W*xT6fH^jYV1w#*I<1L>Taj}`dU+?7peeZO|7Zh!cgtwMr5=fyXLGtm?rP0D*Ax< zeZF7xL2Ilu7fM6N#@LrQtEi{`YWPrh1qAfJiQ;7m?_B^Q4P(a?(X1`}jwOK0^X%|b z;UHul5(%4gi07*OkfUlH9+PYxO#_G}Don(4p-)8C8H-xmRhW69>kks!%KLL<%AP*pPCWi3_3QB-N~Ff}6@Bg@c2^iVD3Lj~(5c?v<+_bKKCAof z08H8AJlv|zEcD2jlDm&9lroQ29+?_16FFJbe&`Kp0?QgOyi~z1hRl}xT!^pZd)GL| zcexVl-(8ywmlSH3OzYq(?tHgqe>Pw6#EdIW%<;|I5$fJttfo9}kyb57)G+jNyxQtN zmp26CB?&S*2#yAI(_jp`yF{BEK&lq0JVHOrqFdm!MlcvX@9qbYK)J>J$x?kkR-bpk zBq|qTNM-S*~*sX`b-sNb3~q6rwVHwzyE zA5e}c%5|NJkKsqm*!8hk)MoJOV`PjGGON&*Z%kDcYJqN%n`iD%Dd1F&#yEmy2bQTt zkP|evZTQYx>6EyXZeD; zk{SNO=-@1CX3`YS61TQI8_`K_b882AA!9yT5dYT?aiaC9Hx?rgccu}RN!XXqqo>?2 z!Q|1i5#Kpid6Vj--$o2FhK8)9;1RGj9>!>NrYRPqy{pB)8ann?->|1Ts;y{w6OqsN zeqa1`6a5^!129VJ`s&hen+?PpE1Rwq4}ZLMBr0(C05qELmnXB&#Is%R@>Sgb&jZxC8`p10ghbupfN*6sPsv{uO}`u<>-% z@(z8Z1PBJ9;!DFpnbPld{?=+#7qIGO{b_Z<#65*MFTV{5EF=)TkVgu!IB!hb zL)do+pu0jaYiQh}Tc5Ch#b3b{={UHVFiOfd7IEO?(9mX}vBl?V)wSJFv>a#UiNY`Uqm7&e%$gqrPhK-j%r-BWt~PLk;(n$9PP2VF*utbh7P1fw zqZGacYeud>X)&<`KR~@Q&bKT6`zU(YjA>BZVEFr+134)k1QMjMtlz=LaKtc7pYD>$ zRApXP^w2b@YB}uHz!0me&CpjLfm;R()GuP|kT@zqAezD-qdblW%>XL5$5dsZYWfcs z`7VhD+}Q0`rK^f}<$D*~3zQ2la@T&CD(KIK`C-eF(%cYkRl`wtJ_Yii-jax>V__7j zi(4e5fCJxFb1DBeCk%vd(e>4;;FtAvyr~#Uhz$r(GRW)6jvSWO;%>VQE>pG6TC#Gg zL7JU>7lLRue^2J0cGrf{hG31OTgg(5u3B~u!A_{!ac$z~AA8lvt6%(RuTySL&df7vjp`mkr(Lf?MVCMdx|wO4a&3y@Z7F1^ zIF|5{7s01{$m>6wLnDg$NM63I0vy|ZUQk$Hw=no0uM7^nebUQon z@B+!-lY!{njMx4&H4qmv;qyW3Tgsfu|A)P|ev4{t|3G0FV3Zc5n=K%S3Ew)m?)~olzUMzU*ZB#rORizfde(aG`~K9ELBS`W z1|~q35XMsi7x00yhZ>2{l`=^V)GAi;?JOD*!LPuuz(fH--V7cQ+-GNs1T5Gbn{YiZ zT6U?k76M5!Dmz>>7?Om7O{5yC94{DO5{5*B7M1Y^eVGm~$yI~Fzj0y?#=}#r$U8~} z{1I+u@UP=HGaWi)i8QznEVHS1sRohb<;GHYVG1y)$hUWJ!3wH+@n4|_AVSfMj0~fh z$a?qYAEosge+1xt(BvgnJsTk$h`@+#!v-xpD1tyM<@b%X)QP;r{W9zB01$aJXFE~w z)L{VGP^0*?Y~S|%SZI}C-`qsEmm!b~bM#)4EaKym4|KZQ^Dw3Bm`W7VsCeMr?V+JT z17)78Q$izRo#S);VT`s@+D+*tSJac<8;VP8Q+ynvgi7kJ;n;SB>@EgHTBleIC^{X4 zDVVOO*gpNg+Xvjm^dlN2rDu8~$2C@J=^tEkHJ|sMqz{H93Mlinnd8mS!K~X=DSaa@ zY6H8n7(pF#QJfSoW|+;YpM_m$w3@hdSNBs|`qj)ml&D%veAx^Cc4Z_g^K^77@6Wg& zsqSI{!8T)hUBK`Sz`W!E>A*+j7F6Bjvs36aL0@RoAz#UnExz(B(YLXZ#lM1o*W%20 zwm*HY9q19f$VaTIfus3|{uEbQR2$o-49~tbJ1nU3QODy#dW+BUUgK@esY`08>h$a( zAE^Q@wnM1xJEfPb8xtNc!E!4b;2R4o+0LBc6P#u2AVO-T9SS`r=j=q6y(|@YEBE5);6ool%_dRx#h(+^Ap{C zpqzW$U+prFL27or_^Y;8WyWZdbK2x^^o2Je4qQIlYQSsu_uKXwI2A z=&N(Mf)=jDy90-H78;GGl_zi{8Y2PktBgmle2*u6%h0@gk~aIf(Wj802Mb@uQs%2z z6m;EuQfzp1@Nm~~@GkLUh4XK4h1137oZu!*sRAH4KOGW1B!XeZShy@l^%hrni5v0; zZ%6#i-`9|51tI=qu0Wzb5wB)CrBagQW~aoVg=@F%sonhsLz%pbVwKte?}i$Hep!4& z4gy@dvy8CRx+oBzI%+eU$5P(ziFfAjjNiTQPfvZ-mTHjdZ=fz?8jS>&X>i*o?4poE zJB!S>n~v6Vk@8=s07o*va9TPpS|fu9Peuqk)t>hU9gqu=bwzcRTO!$ z2PCq5R}!-i^uqW3IWgGSP+7_pObx{fMSYYt683g^Xt$xVd-9H-&i-WcZSO_E=DxjO z0cJ3a-u?pK{axxZC30#VzL{BWn{<(ms%VhYwk?x{u= zed(;h!C|+cque8c8_sy#6MF|R&Y$?6IKs%7DJ{R=2W{R%2*Mc;AbXFT4qa1jQEqAO z!g_1=#&NEsTye+62A)93Vtm#BAp^CwD|S{RyRmU95L-i11z1c+XWHsfc_wMb1J+xB zYjp-%ulnypQsUx}*eDKnY1Ghhw;t$&7DTH7uXCOT4*%)6$`^FXLzShZ09PRS zBZl#G(#4p{hvt^@K*Ad%7OO2hv?-F3`Tw>AU}q4X5N~Br?tCY2Q8-m_AaVmthWV3C zDXa}6&2$F34z%wQjyYUvnf#FkD%&7ItzpUBLzCoHygXCN~#j zX?K5&Tpvattwt}!)7SGVXBF$zDJ#3t`l5}d2E?GQzK0-U2=TYLfaVuyCjiX;4=1Ed ztbo=I9HHG`ILg6*M5DAFMrXx8`oZWf3i7K;Lm?#i$82~P26jsA1<$Qy+)9_wYZ+8q)m zB;iR(DHLb_9DDoqTcUK`O({2rt*7w}@uh-8lEu8TaJuU(c|lB^SpRvW>f`=ZkyomvY8X~y?zj_JkDu?*^jG>>L;2fk z&=2Dmvx8biP?`ZGHviMjZv*Kzcu?!j1F809M&7&a3VBU3QA#~ddBbcbu>3-nU$jhO<$81@-Jy4Y6;Ep=t9cZ?5sOR5-*j0$lI zvPV3mrWjby$VlPgpicI7V9xX za&pg_4xSKRZ1%APc<}ErTrLJ%ND0JZxM;#;rCERP1B{>veB@J)(r;7qRoja=zQR<+ z%_$`3raCzh4H(4pZ^3Hs7xC8p=pP>}K^AG|E z`^Cza2Hz#WheVi>02K6}b`;0Rh62)jZrkI}XByl}HPF+^*S8FMfUHFZ-&=!t1gXn7 zJNVcab}4i|=LDv5HCYtaW;V3NeJ>ma2Mo1r*+7m}AvhvZ1KD7d%XZGfhTh?YL01b9 z6c`>E`4G#Q^s?z>6y$R9(18A38F4^;jp}LEx)#|6gzS%8lAG-t6Rvvde>LSo`fQ{h*#+Senkz_2% z6Cf8ee#r@tu+hQ(%mf?<&LlAW+|~%h9Ue<4^hMYdZ8Y|0CL0qDd?QhzsO=3mblLBL~4;l%UD%}7t#_~q97lSH>&;vq9<4n|Mh;M=zq zh)&cf6cj5JHCH`BVB#KHN%!jiWa%Bi z!=t>gD2oqeW+**sP*No|iN-m+d%<;=j~BP$r;VFoojJU?+%`>Q;fhf0)Xi!&L#ZWj8u3PpwN73Va;C?aqk`wYif_A z@1_WzLMnWUeAJSgl_g#2e$AtJORzyc_h@Nf|1BtTW#HWo)8FeKF5F3zZg%`M)aOGa zc>?L%b>pdm5hY@^wQ^h4n$#Vf%VL3%N~eXK-lqPpETPsGw26Z0GWdb~wriHQR#|(`5|}lz9FW84qk?maFV#7S6hL&va_-6lwk9T4C&JnLmMTyi9@YIv>=% zFv$$>ab4OVYL%-=EB@-^P4j?Rb4y_+1VhgwPIV9HlbAzXbCN(wla@nQe#|4 z#%o9-a%1qa_AA7$AWk+F=!W)B0IVu*S_7cOOPPhd6R}oIyL+Y<086C0O;&?g0pEK( z1jRZcOlzM7NCmQcGT5AVHKpfelR@n!Q8jY>uc~s3t2F3!+OYmL$@j;cT9fy_Y~|rW zE^3#xS_|;JN91NzT0FN?&1IRuERrJ8>>cN?MNgGpV4fY1h%MnYFcH(h|MtZiAz@D<=Wu$CpNc-KXn_waZmA71i3b~zobjKH=1|(8k5>? z4aqG;yNwQUtB-XR_Z{9T=|8p>J=8#9@4fR4ARuW;Y`$H6&D;L`CuW9jQ+m$VFAjku z*X4ovReEnV#!E)R*bqYlE=cDMQ9m5{uT`mxrozwF*Ai`T-uux-VzwkS&qHIs<5%~7 z-Tv`ZHaKz3+D~3nT>*u=~j!0IHsZudWLUgdPaIx0HD6aeq~rKYws*gTHzRX@i*!ugT4v znZm6qB=5pSsLC|GN!de*FgjdLm!e^be2fviFnF0Hip3S2)nY!t&GK@A$nt%SpVH!M z^HI8hLlkz*;=3%t;m0bvg_0%Tp%FH>t!|J^5*K zf#nHG+~(U{olx&#TX#I}(GS){8LN3ej4{CBCrXhVm4V?k{Dn;kFJebg`%d9HiuH@@ zdalaxv8kK-xMF+lDpJH!wXxH$7rtm1Tym!^4B!HF_%8c2#79K>B*0_-1sXRgwv#(bs>u=$!g zaz;G7<>mZ(ExRp;LS`9XZ(+t_(ENOMYkpj{t0-jKoaVUlW35)xUR^y}JK9$aqee)Q z3-kWlD@~ZcH5!O6`i>OJE|gfx3b9U^*?%{A5!zRNDPmAFMHCIdmjY4eJ?Up$M6Fy* zzRVk{LQbrSz6RfI2gYf#E)9?M^QC~83+qo9TL{zD%Y_IdWF?zs))oRo)V-D;S>ZQA z;Vv}K$d$j+D3Qu_#axr000`}f7>fZpFgF+-q3C7-jKtM!DI%HrI=8Pog^W@W({<435hq9Zs634nlyHE zY*bX#H;WfQG=|q7uY|6|DcvEc4+mgbOW3+CIWPL0f2?b?3_fv?W^e;~(-D^`T`6Dy zq!9;o%>b@`Nq`(_0XHT?4M9LSy6pO+0VompI$6-C9zyn6mQoGeiB)U9o}vBE_;4T( zmKkoI;F`*c5jUhXt%eyA$mq-JJFUAOa*z5+Sj?HyrAR78U=R!SD`c?ak1(X-hq=2{TW zBR}=vJI@W%{ie@YwaRYs`UYgOX(;Yr_-Oc%jt?Erqm0->%ia=mlV;>Q&S}~u<*_Uc zZHph>&R$_8yDEXR;R5%+t*2FC9@bJDZuhy+J!|YniH7eRW54!rDxQj~YgOFK*G0ka zI+&2P-2Co^6*A(6e9!&x^gT6Xg1M-qyuSNAFO!6xEZ(rG+!@8X_S^TL@w2`}!`*YQ zm%?nuaO1a1&9S~WIYPd&x*a^ZZo~Qccr8%xe*MrdUT0lEOFqUEw{qM7>vpJzN(WJ} zf=xUw1RS$+Qqyu9*?f0p9hk-oc|P>uD!JOxzn1}|X~3R?7KDw(dDPr7M}9eYf8KIG z(i_Vlu^Mp?oG&YK;Nl0I>lM5#J`K1Q|7hU_qvYX$jw35A4d5|TH^6HX=eUD&R5w+m z84CmtNvY(l>5^5VuigMO(P$*s0;Aju?+GBY*btl zId4+CeEviUcxb75T@P0}f4m+LprsX^+!nO0b<_9@`z@pQQ6Klv|5I*E>8s zq%Hf5e}fVLacoABg6_f4!RQ7<39R}GwJ$gE-Nlllzj|$=bKe1c5Uqyh50ghWzsf|w z;J#Wi6PLojM$^th^W*ol-@4aoc;^@2VMp>pO3-_^*f=Uz<*fSJ zknnU^mz^nNmkS2?6f}PRVQZY@9uJ!h<)Rvn$a6X4)2Zj;pxg)>(`cLPO9!`@!;Ptd2y}3FC(Y;Q1@f&VO~arT7}$u$*yp# zpy?0O{KSeEBONWxB8bT-L^u;iIm#17srmxiZSPo5isxY;NYzA1yno_-YK8qG8*ohI z>~`T#Y#g2mK6iSaxQY!fn{L7vv3E2q)!a4vEG}f{rBMKETLz{`j-jO~mA9>>R0CeP zi%zqe(rm$~^kM99L6hKG(pJ+9rZM0|r4P$hrFit4ljQrrtM+;UKd})3XVP~bkr)Re zYS<26)BpZ96SyV8$JaJTK4S^Z2j=Pr1?6-h1V1?`N#-I*37LL_8a5`@aR;a^YF|8l z%z5@5sck1hLiFtfO&(GJUbkmN8R%k}ILTQF$O8uk{D(B~1)p=>L_RPQG=nSV!qCBh zVF_%F3Ss?Bf;5Q+EbLRPM$U_GO{xC|6Mu`fwj9Z8n(IvC`LBDJBzT#U9MFZ3)W>b3 zaXL`YuUjW@kzc`rY6qQfS*C9z3Zr}&7`S2;{y=2skVCNr5ev$?*zsxviPT=u?wTR< zg`9>?^TnA33!mb*yFs91@2fUxs*P5!mqy1%3^R@=%xok3q@FaX{l37-&!Jl$4teH< zvJ$@z2KQ^qEbkB|>!w%2aDKa9FhY!)SBxdl%Mmlk8Cx!aVF~H@`f`S3!2ZI?vsp?a zC^as&O=RiIvuCMKxfBD*##&tXn!8uYv&!nSz6Y6+uT2Q7py^vLno+gkh-|tsv+qN1 zzm4v<(E9&++bO?ZZo6h2ddYKB+@>LFj2X!KTCUEp9c%q5{b_=pXMPIvHUkiO340GK z5ko>6opy$AIvdO(4f{R311)&f+A)VX@Jp0*WX@(g5OZz==WCWY3p? zgvXajUgJSq0>MwZcw1*Lg^%~;uL1SSM+_v;kw`4grx2hK6YlZ`f@vV-pw{#tO78;^ z2R30Hd;ml90vPD-*hguDs9PQnEOU|ma zxP@f4hifl97I}4G$%`oNSNj?1Xe9rh#Xhu-G+DjXxmO3|pscM~C1+(|}_tWrP-PYOSMH9o5?Tfuk+3FH!ITBX9Jl zB_wU=@q_7Dm1`&Xqo5AzQj-6YE6{?_2;82G{ZKFnt8+tm(r;#xU?7&!ar8YPGJr{( z+M09YpYDJW^bQ4C=nmqfcKI&ybc=}`*WK=N4b(i1{cflpGBQ$1apaNVi``YSq_aWy zv-`3k(oK2Lzy2iGQ2$X-G_?!l=yk>he)|&I#YB1n5!QD>Lv{GUkXtxCkty3pzGr1* zj{b_jQ?}Ad>f*mYc`Z9rt1Zc55^~_M=NBW+B!$Fx@m5Us)8=peqLy!*!+r!wQ(;4H zWB%3bKdH!@FQFUztvu8E*EXHKeBPwOpiGp%GeszBNk6^E4I*OY=24Ls1~ z)XzEZmG{kq9YW}NgKiIMD6Zpogc!Eqk|B%Ta;sP;qk+D(n8Me-HYLB2>~Um*U0@q@ zR(x}4t&LL{R`gAFqV9p(|yNp=LKwmTsLk zg3JNfPUcIcH2u>NUCPfKPGOBLQy7bGJB?+6;6s!ggS?m$oNMWwkSgj>ZwE-= zBORPv^8LN5w$p=+r2xoF^()mMUVqf8$@YrrH%QWbjPd6r0li-RbwV|bC8*7(yTS-_ zGOasBU2l3Zr9R_BwoZ}%`(J%aD<6P&oMSWS7i60qH#>7WzYIi@o5EoKS*N3RBWwsi z@|5i~WmGip@|vxJ)+S<)EPvWgciZXb3B7E3LWiVvokU)V)3zB^|Hfw{Uufs52-6B% zs)P0AYLtVVu%M#x%y921V;5(6&AGP@aGh){tQQMg1GS)eI_?FQOu~<-$S)*uk>2)l z;Rzq`UQ{su2W|%Lzf#--hr^_=tE-(Yg~YQMI~FB-#V4*H)$Exi%^bCqJ{qJXg3;64 zyPmQqG&)kd_-w-)RbDLg6of3UrLO!pd+>lt7tjIS*}InZ`evC(q{jgC|Kxa;Si26hG3 z|04bVq({={aFr@99fRtvaDq83E}Uq0>Z)lv@+@y(esRR0U;h2(8A$`7l$Vt4;d|^K z)u$HI&|j^&wu7G?13+7+yqsio);w{J={j8}uDE=Y($Q`rr3^d*^S)OCD=55i-s$2~ zp!{=5*ScsuU5?;VI+Mo>y@-xtxyAB z{zuf12Y%osAH7tTkjLYhNaosX^B7Ei0Ko46VIYt$BD5g$@XaWvABR#U+wEibcgNgKPCxkVj(B!MhS$(%%#zC3T|pa z8>5LEjv9_JTmaq<4W-MSbhs+z_z@kcKssqjM`}?tD6UYGV2E(KtBq7xXRqI6;G~Np zbsW0j6`^sySW5!s?|gPFHtp~wNN$um8nffS`M74z3bhmz_PvIO_?XD|aN0hD9t{RI z$dXpWej=XaQ^pMMIK4avM`kY@KSo7>RncoyUwO3h8r{nm|p-cN21ddRS?8lB7nG8DQhxRbxwx$4J?iXk0 zZhSX~&rR$jb1?-aB#V3^{JFofbYHnKPfCkT6;5P-skiGNDYeiapsm9xo&p^^ZFlq9 z71bG6j1cs+-6gT-g#GdhEXC@1WNxj!yGmK0TY=Y^EspEGG+*31=Cd4}>O+3*V5@+Q z_*D*&q21xa`E;MZeN!MIA#nzTmJdgx$u#*Pl>Xu0WN?s87i4kznWo0%={DFqeu++^ zqIn(1uv&}^<*pIaZ+7s`FX!buxY?Rq_-I&nljqrG$ z_f@D}A(t)*g!oz5zRioyR3B=H_0xAl2XWv(7b>#$IjDAwcYKq}M(FU7n%xDOn_Au0IiQ)Z~(AF89gWKM|> zG>ZAWx!q@uCc>miYtTb4`e<>ro&W4TXIy4T8VMgG^jBw=-|aOEz_XYaMb2yI?Goog zt{vyGN*u~o_o<|(r?*b#yIKGWSP=t-a=|;r!UDZAxlQio_2m1-KCMIk;w?e5-J;-A zh3(tQBAW-SzO7oS2fBy*HDc=w7Xz&Dh`qsR*iT{>(`*43Osrqh^42jbC~*P5$-nv1 zI0y2=GgNnIz=oQFzdX=Y>YjGsXq`?!d_okR(0+IMm5w5FUB{wUu1dvmutT?okgbRR zYf7V#*Fx%Et_t+Gji*%nlR@8a9|pfu3zv@&z1Fh(9|-+QgKN64y`=j9jyZtoWo+hu z-*J3mF47@;vXx~0h}&RDt*GXpk&9vo-&45k$6*fx>QKWD!^)b~&bX$=0U7(T(3@$g z!kiJJ9kVdu0>+dK8>ttdB_6lc)c3kE2v_;dxp|-EM-3qS36y_e-v28YkO#;NEIe^} zG81H{f<77HX7yT4E-C;O<^-b&ibr?^oOQi0DYnMNvhs`l8sOpNw#UJq{jum{Nq>@# z*p}v-O0#=1M*uu}fgaNEbsLqF$mw?VBi|!TKDo(^k&utg_}QlPKaiFwc%F8eEw{bb zYyQ1C<;Y*#asL)_?V;V>1%tOv8ppar)eFyt*N+{rO>-fnHhcjzi*7gSFVBp3f@Y_0 zno6NN@S7k_en7MF;q@j9tgV}$N1$s$Lbe2>(0_hJXzZwo6*%^HeWTpNL$S-%-=eIB zB3TT8#(L{)Ss0w|n>x3VQ4elIm|Fn|^_OZBE5>WVC&!mfY%QZah*F}lE3tvr_96=F z4u0V>lV6>3F9!CC*MfxxdR1O2)E1B=1h~?4nA-H8H?Q%cqxraG6BwUuga>dC&d{_e2bv3(w#6_2sIThrPml8bkt+tAa&--j&y4{UV^gFbzr> zdYZRsZvf*I5)uNfE)@By<2}aQaZetvP9ETMZj(636B4t$jF?S)vitjot9K5p{@ljt zsjkp_?CZ#Ld&V8jGR=%kox^i%1H^mP8$uq8<&Gun8YOiz++}^A;Hq=AKm%m(q7z@6 z9~oH1TCB|0U#XRnMPRCfm+dYtNYFqw5JUU`x!A!#`195%VsZ(G!(tcV%tWvq;_`PE!0&W|HaCi353)tCl-YjYohqx>U!4x7gvbhpC!eN&IZ+`` z;j2IbvppJcaniy!Vj{?5KmSnk>*@Paseh1xKf5zwL!dKbjrbT31s!PG4vDNSZD9LZ z7S?fc#H6c5UiD92t!||N1m|_(*&w0Hnp4LIYC+U`OL*l*W5OPV)qGES|}>P;00mI zHv5(*(uEl*UQSMvgYfsvq!#4cq1CLsN}zCpd#vfzH9 zLORs8J+yyTCN3DNUq*8w(ZFZE?Oq|_&!ePzqjpv*I2f~8gAuDskhvzmm=`p+Y_NoP zf(4dE$$iDM)rs>xsY8cc%T~NpYB6y-KRE1mx-PMVGCe?Yn|-_~Zx~~Rj%p^IHF?8) zWLd=;RxXzXz!?NoP8XG8Wd`2Wx~cBJdb7#dxI$e?)|U~YlF})GYVW_;H7QWmGddS| z*yGgK?#uE=Q)#Sz%WVrC=7l&s!c8yJD>fUClV5Ria(X5y(l6;Fu^Cv!!0t4QxA*8; z@Au}7b!Rt~Q(>DLdDsee?8&o+w2WUr=Rb70sxRfIh2b+}aD0;Q`}9(%zW8Bw)+d-OH(AGL7*^TxJVU>; z-TTwKPG@9ge#GeR#d%1J^+#7>g(JG-3rfnDGjR~i1AW7mZmF=cQlSgIj906l>=1Nz z5vpD>(4e;imF(wzf{0?q#!FQ4{5wnRh1XR|^NX%S-&GZo(Ij!3Z+4%xN*>gWcH<=P z;|n1ui*q28khN;}DLtT>8W%l z7uVcU<5DT3Yx#`J23F*9)+Lux@VTAJ@qFJ}d~KGLwF{X@nVt(7EQ6u*|Z8SYak zm=!m6T~eclcdpzHxhSV@@kTajDRlI}XQPI5Z1lm{u4gSotDA}U@Ocvtqv8-P03||JiHoL;YXv?|mLQdMttDv1k}RpQR^-M@)S? z9iy_t?5yw&O-l7*cv2x3TV6`mI=AyG~U4@-9O&rm!a+TlT zK)(=Ge*5-CTb0r2-E{Uu@jnXizrctK0zm$FJ^nk7GB(k5Xm+4B*Qu|Jug|m%&{q8s zQD?Cc3l>8B`2N{5l_%|TQ*<7EB5yy&1Ltv(BG4nGSxvncw_~Bu&byr^q$c%3Bw_LV z);t_7H=L7nnIR;lwA`HG5=dZw3?|?;e+;zm5p=!gFtKS`8NPMpnej?rb%l>pilZaX zkM(9Yt^EGG)HVkKE8pz+te}@4n6(Ej$35X6*P4DvRPF;pC`|#Feu^TuPXs`KH%Nfx zsFiy=>Gv}pP}HuDw6dTddU~yug z6SGLp(-&ji{Kj{|r6?CrrpU+(J%<@Y)Q&@J_od`LhNNQ>+zC^wEKA!e%4s({w;Qm9bF~(A++!Kd(PJ4k{iKw^W#-tY9Ja8+7$6)TNZ{aSx-cSzAp9~y4F^t z!hXz9((K2l%g{3tKY$Sls+ypK_3B9hL)FKwbML_V4+h31II1SHFyWInLB{A=b0tF4q20M20WkF} zpd{962_Ha#DXGMG*T1~Q^Pn4C7<`Jpm}~0wjlu00P@`eZLvOqYk=87atkEW8ZX-hl zbL2yXa;-zWnS7`scb=ceznM0tz>T=GZoy)9#00b-c#oGT<@qTj>{!Xn5t8Tsc)$J> zmIgc)6GoYQM|?8jwDe(?ib#^1-jJ`UWrgw(#AC)zyaj2w1DO-!J=o7=i1dGS9@!y+#3>`i#Kus2Ql=`kuVs#nZ5D>OORNx=c7? zWXE$=bZ!mqRqOyslsI+S@67~du;dKIvn^` zzUUlru&(NTj*aaUO->wT)gdw%rRs7wcp(;ZNQR29p)=lBWATgZn0e}Qu8<9mGl`XU zE5K$R49fAF|KC#l^n-uWpgO@3ZwUr+=CWX-qi`Q)hc>`es8j5UciP1mTe!8=x?rXL zjIb%Vs3BY2?Q)rwP1P}j;C$0)OIq}C+}-MqMk%E+&^cbR{XQY|(`*JRkAs=_E{{Ym zUjz8=|e(t6`cA~vVNj|N_AQ@7Fbm&gBrjWgoyavWM+9F{x zSo@TD7cqDT4H#ajR2$4tNJkya0>)f4>9E6Jw&%78)6I$uw$>|u=_$cX2UinARoTnF zO~6X zVYCC!IZ^)>z^5qHqESk?ZNQ>|kmPCJzHI0I@VaNrieVl72m5JjR=aq;?59T@fr%Az zW^)N;)s&dDHyO*?*zC?;U!B0*C?xCSC+5`vjEL7oK{}`J#^0uO6Vu+9e4JfHF@_K^ zoExD6NV{YKzn!3xJ({8eb0>S~Ss6JJdouty8Qo#gKQ%i#*bk*@W{@!~M7!9h6Qj?>fVgg*@Ga*U;CRZKg?FjAU#{*IU6u2PUr_7HxQt7T>M>@P~oy$zVL5&!6Q` zIB6mgaOUu|0D=q3W6|w-&!mgZWa)FLF8UcYY@2oFFe@cg=%#M@+%rWTQ=8dc7FTOY zFtBjd^E=>cE+c?OYW#X1866xrp`X6ms z58t#Fu^A!4HanWuhXso2km0F6gEl5fx^o)2N zaX`cM{P$y<3I!q<8MYk7ZqRd1Kl$nJOE-Jb>9DEOf{4?dsP5nHCKmz(?ax;ycm2e^ zH`6^M9i3n?!tfUu!N^jOz8!iZA&}@Z9`x(Qd<06{2bx_vX>!`%tc%C`Z3Nc`EZu1| z5OKhG@@7kdJowzl1VmQB^+Mhj;!tp#r$z*zu z=nqugAYMbGE`Xdhm;_whugOPlM0~XMp9K=T9e` z4mLvcIY5<(ude^V6QM+2#%9laMCD>pU?>ya`aZp}XN@xp$4mPJi=~NetR^iqpIdIC z4cCY&$#*S-m9-rP|Dw$%Lh`+ct~F(zB0bN;qa9>pjW%w&&f99L=9?oVWVqf&knq!a z;RL0TZDVC<_O?-O*t~@$nr+A|-u^di=02`sOB5KA^k|?!2xk(X+sg+*GLX{RR!gAg zlK!^-d6G-SRl4o|5Ek*AG7)t=xK{BRAYUhXkixEW2b%Mk8-0&&8>6sJJeDpE*p} zCi9Du)H-Wa6k5|K3Gi|dzh-ebe0bKmhE@K}hVn-|VnFe{^5gBtgme}4f04(4B^(9J zVW3OmuSHTXzIM9D!dilYYWxg-ylNtvAJ!#;-NsIH{9tlsf$UTa z4#7*PLRf*bxkCOM0yBTuv?CO&cy=~6q%yF)diAR`ou6uMK1h6JWQ{C89Wm6&j%8&v}}@BcE~fBCaoqku2vD*pchAO5Z;Y!4S! zj`4l$?SC!ZU;nKH-UXOw+yDR1|9!jv8~^zKZ~p(M9=$%&fVnvxAi|9B;Q;PexwWe?FJj35b`C148x4x7xatmMc~ii?wY$VDhf{Px%0nFoS^ z(a}|EZ08x-*<%5D79$78SFIXbT~sx-(*!_{Gttsj-_8_{MA-<$C*g$%0LS>k&O@-s z)J7(@e*l|ddLrz|L7I2sl{@B#p zOv!h*sOp(d*VosTWV+tS49gWs7EWfzbAfa{Ytm(%7?d2>-`+9rkGM2$2l}V!BLTkp zjPzE|3De&CA!Si&I_5a#QRC+fT!mjf$8C!sDSXv94SMury=%9VD^@L3n^ZrCi*g8v zm%IoqeaS27Yi9ImP=)dGenNn7s3c`&(*qSM&5|t@G3((K_GjzMhElU?tP3QfI{lh|BLN4Dl)^b7(SV`BrAG1w_S_;S|Vp zequ;W*(-FooBE%&C#)m^C!EJTXTYb(2xCW(j=S;;y7vCP2_7hkl8fzh7)X7}HpQa$7mFl3*VoR0% zPCxN{R8;x;kThUmcdT-+mG9&_1LygtbD)9I7olz~MUz!81*}2l@r526Beg;nLsT_y zC0)Zyg)@%-kNY0Lvlw%S=)N*|je0fp;<(%yX;G3IFfgy%)Q6sRzdS>H+}K!D)x!4ffZ*=ZxJz(?ySsbi?(Pz3ypcfTOh50uUro(a&975X{e$A1 zK6~$VU)NgE$4%_l9!weHa@5sb0khnMquy+k|M!YzDE*%ueAH|3dfRAuBT=gfs9%-3rU$U>GZt_Qk*uI2>+3mmp9tNprl$zTM##S^2rAb8R8 z!SOlzEdxX*4Y$p7X>PP7MQBv`zdHBbdn9<^J(HVY}sV5!@~*({t?ug%NfX2W@R-qF5ABX5emu_T_wKT zES6(dk^>ns#1-IHZf=OnYT`%$KjnQ!%;2-ZFQ3r)6Qw;bQI+q~OeA+!Uz;go>o@lV z;fb}Pm?Yk~F`U8`I*YzfQZXMSCf|yskq}R~_?surF%36X=8EXEiOclX0}yS@r(1X2 z=6Y@gzWjL?C`j{D7!#L%-IZ|h@GUv~;9HC3m|kvqXd|YvWhvtHkKorriNsk zYmTQ!5h#GCh8SNPvnhICqFU_WsThk|bqDVH27T7SOyNMV^G1!`5h)rdSR^Lt;??U1 zw3wwt&PO<_WzMs#$Jg?Onr+$vAMLZ{%zW_;F}#SNg3ov3tSJru;s^g%X5Rt`6*lqq zSCg$?YLPGAU|MT2rF6;oY`FjCjL+7~`kO=sguCdIRCM#rI!-?m zp5k5kq)11r1M_r2`tCc$At*5$MBK`Kl&hWR$&7x9adq+RsBoC5Dk~cI!P8Zyx z5(r21ucjKY$>es}-^Fzs4#-Kmxy1@jY$fi*^nSZrB5G}F6jXPF96&U5@)4`JizyrA z#c!{Q^&>v78@%)#hY`mh1br?{>!XZ%ObUFFmloMjh!*lyc2z;$d3E{oPMX!bGF4Ky z%Py0|y=iLo;HG`ds|_w<7oSg+jc;6IWRgI{zHX-e@P*$6H94&T(~+{}WgT-was@3?Dt=D7Mx_;aSez;ic4{qAK3vv<%-4-gc*(Mz|Rs{Dk&r0Xn)0 zb)QV5V80<>upIkV<@t2=x#$2argPP)-Be1(^xehC0k0Q})BDu~-cArrtCAcXl;W;` zY^+kf6uDMRba#kKxWd*VvQWALH^N(%@}#b#h*UX@(W_C8DKUUMm!WIXW|&RO!t0 zXtF(E%8gW?a3y+rU`S(M2@a1{=n90CdV1Kz>N`EBKf<8OP=f0hTiYg}$SDk1l})_G z*5qkl_XNZ$83-t%o#k6h$V!sYF|o-i*Cd>4u|+eG_g~Ln^w_5F^a!N-;et>Z*V?R{ zx&4j(9=RscYNF_^9z2OR9mq=QCjsd2mTY|ElzJ$GTlUN$)yU^4(ndA3%!+#r#nHMt zA@?G>5;T;I2f92tRXVOcjA(z)$x~k@NKS@cdFF6V245+jx_QltJ~Xv-gRFiHvb>3L zxfhp{i%fU|TbA%+K7Qjrm9MS1IVN;Pfp0)kHJ<#|?_YDuKa!m?6b~dzRj(FS)2ppQ z!Ce*D*fz_5a0WjVL-{w@$GxzdL~9AMc-&QKWotc>Jw3DbgaX>82?D37v!dM(wqJAR z+6kcD@Y?9Llb}LYy?a$|S39iI#n&B<>o$Ku^V#KKd}h!yv1#H@aI@vfvci)XGPx(q z%t(HX249K(LqmKu_8r<;Im6!hPVL~xNBL?ZAff20NGJ;X#(zi;<%`1I0hY9+D$ol^ z+`{6DtpXt>B1*>5Vb?V>?<09TTHGkmnw;t*>R7c7I!|jBvaa!mcS3`n^UK=&1JTeL z%-y$<(WIw=W!3dkea+2Aw)ocg9I>-Sd8w_l_nRd%|nBwN1})&uMf_#xR&#%6yw0WS<*tX4UGUusG0tJs)yioUa7FiVW{ zO#DE7_o2~ZdTv2rFmOcpsv|(H8l6Y;yapj_8Wrx!XOeYf3x@Q92+~bpIDh0Imwa5$ zJNedd= zbx%8_Djag`;SjDGzsGp5x)tn;58`HsfaLoB8p})=qde zoVRTZ`HG6?=F;HBbV8FZ2)gXdt9*D*geh^oIaU(r#sIV$qy|F0MG0wYI3iHp?j#({ zr8={lo|KUkCrNLkPg+CWz^7Xw*G^A#L$Cx9rOVFk@Kt^?t!=TtN8cme>Ygc3r&{31 zAPouf&y4^4`16o87I*?nlzBt4^_5s^lHZh=i;%K@l+HhParU2CS(KztlWUvBe2!+{ z7Q8?kE>~ip=CtIPQ;MX48(>R$C`PG58RvWWLZeR{{E2t%hw8)SjUlv{t^%)q?UGa6H_WsJ&Hr zBNykHXY{G@?_q;%Ki)N`teeBbSjH$3UMdOB>o8nmrqR*Q5F>xw5f-@P^r2gg_mO#c zVq#yYW0GSiA8?<5_X3t`*B0>J6n_pfJ_H%duMn*9H}V@_T!|T1a=E4Jwaa;dT3~1}Tj1$$Mqs<)2Br#Q17q6(9tA34hg3 zhf$Jn=RcD8Y3AualK97@JAz_3{3K+d`i}!;5g79FN8;yb_KZF}Z58z+l0tCgdyH%E z{{Qv(kyPF^F+G?Yw|F(A)7{Wml!D10GUFv$R?{-!mr;H2zrBaszl~D#sD>ju8KG~u zSZPXnn!-+QW~=96~6{Le)J;(DflC(cl2fmarHn9W z-|4obuGXFaDvZmKO5whczAUw&bQLM>!yUtzho=+>3^-yGf1d*N_#&4*|48Oh;@f2# zlELE}Dp!a8xj-OYo)e}rYw}6h7zt}4<2zw!Y)9#{VQ!L_VLYxT=c^z21D{p)yNCq@ zUQ>9lmiN?UD%2(WScWKADSC(|$=;thx#4Gs&uBmrv{kBmN-5TjG&0_4RXr-A36a}e zY$cUg7s^0mB=9l9!P%e$1$b_e(p`QMl#9j#j5al21klw`^A;JEL2Mo{|AZBfwJt`Z zDfm3b8vizWZ>5C)1+8evDs=|2F`7#5dDVcP!x&hYLy1$mr8_BQg~cNY%&7n!=4>FC z)vwei8TlYw!Eo(~IX0d}{kHeH?kiPbZb{OkxF08tQ)O_@1QXJ^kpZ*?VtAy-LdN(B zEzxPax}vb!aiUmr<8N`C>LrHy0VEP}X`gg(X(~xdC|RxiC(3m08UO2PH;Tf zx5?*#P<1bpkk)XK$|=PggXrH9sj$zqi-B=50o+ERXr3#4a{7;5H4_X37 zIdJ&*Wc00~1HjV;KfR5QxWbU%1hj=5RBjxtTk`XShFg6nDIl-gjKyafj6gvBsGSP# z(|$80oth^-90Vy!>bXO91F~qHd1ALL&TTeam z2hWYW_MVN$5yB#^a;_zWdlC4KP6+C3vwdzdM6@-Q1E(C4U~js(F1`o4Xa_@*xuFL! z({MO31h_8zv-G5oES;^`qWk6Ew@_>3z~H6E_FFZkpZAiBB~|W7Aj=^xsVYaXI^N5FRsGGW%>Ya# z)u}TurEM<0wI-mM<;gFYlOkZ}q4BChu+!s!jtXP5kTKW9!7q$)1+FvKhrbZc!^!d# zqlKgAzvy$nqwH37s-s!z;iEo~>8-=ykc&!ty-s6OCDtRng=0v0fm6jlU=M>pU|^;{ zp3E>|T4=OpT{ts%^B0x!mMc2rQ{DP!XAwS;>_*PWzk7Jp1%m};Xo_i<0Rau3aDl&Y zFfmQ&xakQGp`NpDtdpC@z?xG!G-O?EBZp^oPzL85Nm+Ot1RFX?n@#tet`fm-z-Rh> zTX$PkZXw=bwjCWDu901Y^^FK`{1T393TT|R1TVVLT*L4J(h&4(oWWvIxK(@BbXXy3 zVaoq9E#R6RgF073!(HG35&6`v+LTtxDB5gF_f`)G#y^d68fjs+Y3z8$4jv~6l5`IF z27vt~MLqg(#+qsfysa{#9=o(=ydG#fxqr1ntriNLV1)jzw6tpAI%p|jpT*tXyLeQF zh5vkeBep?r(!S4iH-1LE(0C#mKpim6K{^jUJva7lb`wk;r$a+Fjq~~fF zz6GfL(9=Ty4#fx=6ONHNW5p3QbmEXYgy#?0%>oRpjhJ#yH9p!6-v zn7E-x-(G<6M=ZnbazRq6h9{$(^_h{x=iO$^GiuVe?IHcEAGmE3Qg8aW0pdL)Pe|v( z<3dXDXCP`?Ykxaec-``7_QfvjwplR}Q|E3$(i_kjZjHs`1{BQsEa_vR67dK^ROScT zB3F{%Z+cn+w||4IfPV0L4+SJ46%p=@;G9j-;8XzMje%>7{At1< z^>~OM70VCEV1?~Yv!E1-=QRUXKKv+L>fBV3s_-P1A)n|+{HpiE*;&2Rk9fy4A%kBQ zjXdZc#gN>>opjew%5f^ddON9eq<7uZY1#Oke8DalFWof1`2+Ug4_p!@V#{}0U$jlx zFeF@UrVeP1%d`7UlB~GmMkTfH++9|Hk&#K)7%-81zz65!u#L~wE~Pc_VAtNk!PAyw z`Gv!`izdvRGF)*kCV1LiwQ<)CmPe_I|FnurA*v#2sE9GwE8re@3b>yD#EpAZ>Od~g@1%V4$65hEJ4^yC+6h$2zNi9-@x4N6& z_XDfeZTlCX4%4agaFwZ?`T9H|MV3i(0iW}w$BW^ zR8i@`G$Z>xRkrs%!mOnQIX*sFy62rMc!?C;oahao2|9v07bqc(W}Y2!)!hTkV+)xG!cSY5Ib&~_5N4Me=&7Ld zrvxBQ)6v`|5rPZceEJ`+cI8xD?+*TmEBiQQz#BrY?up2H*#sPbK66v!7Im%o!rjUs zQUah_pbNb_eg)88HBhO$sLRJo)Uw!Hho+NcJa9)Ypu0KwfUCBs>!^4-xptZ8SqMTx zHLA2fapi~kZoLOpe5!|H)e^A4GWv?c#c^RAl;ioMXn0(ZGx+{-_xmfC2L1si+Jzf* zJeil%$~(nGtEZokx~^r@BJ{)#h^6-hfZak103qjQFQr7izoOzgTepdC0)p43sh@Zt zPQ1R-_8L869ZU!2Gn{FT^}+loQ=fs~E;4#Uw=B3bZ> z_zA}^bq|?Bs(hOs)6oqZUVIGd#+#J9tM7BfE=+i$=8Yfd2=9XjL)Q(P5?M+Xj zXrS87Ua8$}5t~`Jk2Qzq-z_(Xil~o*{7#{I9mq;;eDn9wCWS3tw$dJ?{t!hX7%$;5}EK2(UDzSSI)37+>)rivZ`cari7W$+G;?{hqcIOjrr z2dI^8fARK8?^zj7a*w>DIWQc6$c@!{KnXQ?Kih!6jiF9H01} zUG(-94X$dbRptS|q05v_T@PHpXogl`cJ_UE14i{;kcSjQr221n70zqLIiq{NihHM$ z4T(2^bt3uf+`pWFJ?><7JyT;z?N+xvX^KLONjoj~d@_G;(1*F>T-#Xm<;XVq>LXDA zhJ$iY(}YER=5s%I%Ye{sb2a(3R0IdxGRMJUnb$whhGTI+OPhzm+^-aJ-W&l_Z%BGX z>n}P{w^RYX4OR6e4xI8%vDp$@%LbM|UQlJcm1&ML$+-FvMQLq#LvbhmSV ztaA3AIj*$`_!n-Z!(hN@o24D#T8_%YzIm>R#M&Zx_ImF8CA0(XUU@+H1->(Jo6 z17e8{@|dLGGJ=VqpS;~N&Uhx!H7b~@{alS5eU?&JvYtsmnU#n+Q(f{ClYZ?=by*B6q9t zqgUdqJZNP^`bcpvio|)zK8^YswerJuS%6X3krS%EE>0}m%1+4YDE#g zw*8kd3Zh`b+Yz%9IH~DGx&K(7Q4^=BuP?wo)w`_CV&x30%zVxe7=kzm%UN^`y7&d* z4^QCI3B>*p=2=id3d$Sgf&tglE>1(&P}CXQDDq2Q#_!>d)!)FkLoMC3e*>t7{?ND= z;ld&4Bt%3H!@rCx%)H-E>hq>Q3!vbAGB+1;(dmHXTt7B#=bUG)-9j>aC^s>BF=LY5 z0H{kkh}*5mHo`$CYHYHDcCLK2Xc2r&r%qh^Sxt*>1;w7SnNGb#CoCh6r6slRA3A<* zvOZvZ4ZJL`KiK#)7P)?;LVbD8@MmB~w2fjbQ|8SLtx%)5|Jl&_L&fO+g4DG^QnjJO z66yr15-xrNZv5O1PbiV4t6Y7{^RMST{1P&?6y*AisYR*odOH}0fvz3VhSkwhj}rxT zbJ%$-gmFE>K6|^btpQz1CO!}QT^na62lmGd5Zvj~;2gpV`@BjEbK7A96mtEB`IH=N zcvY0CPt?tKXftMvda>64WiyCP(^dwhF3RW#y_-_Y;H`7;=WpSphHOu`!;t#=Xa2~1 z4`;M+k?;@4dSq_JCT>2zTItj@@eq(h{6+6H-Wvk&wv2XS$Y9luvo)cD zH~Vz603iDuoR z%iP-Oo9uAi^4WqNba_pR!7_ZH3;ist&NzR&&*xs~6!(cO5ur=vo>kKE9i%&+#++t{8Cc z{|HJ{~cyv^-z1RUdXjT^q_;C>WI}_Yu)&%WvBIC*tNG9A`p>EFC*!xxGwJ=QZ54=}T@g?{f++Z`X#>6{BJMWPPLK$<1gHPD zyvZ7kJ-~)O$WG|4SE%EQ&!EddoChD6%*H^f$iED_2Flo2=hHF?zK}>;B}zvbNEr)O zL#3~Z$xNPDR6k~NXf4r`dp;fn7Als<*NI*8=ypTDkBHXlrvLhT_;5qOx9bw$F!$wN znyG0t{xtLUZKEB(PJ#~c2&e*^F|VjkFmct|e?MqKv{v@puTaL{+d2~zZZp>z>cJhL z2AfsAsd~k!X4!l@*er07lJDx44zq2Ut~GLeiykl5n_ZsH$gHx@qn$eEj7iU_uZynx8*kd$0B=S7%+W+ z2}CH^LCJ=H+E?TbAf1BSrkt?ZLg92eaqq z(8x}6x$SWMC8VFu`-{outCV2YAKK8Hdsj=b4WWRuy(T`Jc>}1SM6_`Cx}!+@P3~RG z*A#jHH1<)>|>puj(2a zeP>$qYjLk)l?$XIZ&sZ<2&Zjn8CU;2oUc?0O^nTHg+BOfH;20cH{&V)b)Id~Zbr&@ zp{cgfuZVP%_xa*hl&=Zt=<1}*=ahD{Tdn65@>;FW%o2Ikm&nY@wOqSP1U9-XXBEh~ zH<%CA!4BKK%VBYGTUO~N!x1u?9|%v*!AcViH}?8SU}_`M<`~XvNl4h@efqc^c{kudkdY ziL7*gA@{#%jy7*aHQ=+0Nq#5QPYDC=vb60#ZhpIU0pVlFvHuPCF+MLm$XnBv-^uK# z-=(EY%FZjTnXK?~+wXG>UnJZAu>24;zIA2Z>a#B5SjN2MN2&I=n7w@|5$H~X1@(+1 zThVWIKi4|-9bjK&%5rOY+;KZ&&`Yu&j+DR|ksPH~`v`6uDZKF#KKH{oda^%JeV z8P)miI!wS9tzS#7KmxsM_ij3Jz z+Cfztqb+QQcBUT7sjnedC|0KRg3VI`){mz9Gi;^#QHQ?euLk35SMol#3Mo833-q2Z zJe$;Ft63c1^F_y6c8o@#de-Y#LOyMgO`ePM7PHgxItw^}aMJF*tidQZ=w>>(WPabJ zZh6wyNL5UQA#t-sP1FG!c2qnOE2HDw@-yYIfQ4+~Bx-orM_IxTV7T zT4aH5!LIW+HGR z1WuZNfD_A{8rbd6TF>yn*;SQOGq9tyw%%dVvWoC$>PCK>)w;)plb>Ds5?4Yz2hVyZ z+nSqWzv!pMQ59;M&KG2x)4n(z8~MbZ_#>Ily~O^_Ou|K_GWgY6B$QkxiB*wvMTIQD zLRC&$#^@@?e2r6@A2nn>bW<4%2Fbre6MxO2pLe!w$#19^(Q~W zO}!h+Kge2Hbw+{?++9tmXhD#JRRzv|MYx3ZMg~oQ%e=t-F1`T)=sK8E-~orS)}r156|O$a1dHWeW#-x=;u;FZh|KeZDTF$?^^cz& z*5DC8^EIpM^V%&)J!IZU5EWxu_{O!z?l$0MYYB4Unbv$CdyK-hM4FQsuw!kJ!eQ~@ z>{@}--sPjuf!%X`OnUXw)&_1vl$9b2lyh9tLB81h8l+lalh9S<9WW4|(9Tv!F1Ig# zbC;fj+z1`UXfFnz3&{N{#<>r^C5}AHqGb{qmCx*~xMTx|39y~Xfs9e|pY?Dj|7~HC zq6H?^OJ8Srp=61l%oQuX;-mn514+eVgLvfDayxc)eXZ`77HITHuhM&j=A9gPz1-AX zH-&vWZigJ#S_EvqT09)fPu%eqjNJ^PV2GTH{rIpgM7g?;`X_5w>?OkhwsQS%G6L<) zx6i+UWskn|KvCI2p6tS)$6{^x+>nceGg*o9iASSDz}&mS=Ec@7-qla~OrrvZBexlN zEan8_?!D(f^^;(rCOzy3c)Vxyp*|>u4L^EzrZF`qw!oGJh*X&=Qqz%5G#+X-UBvKZ9dpB{f? zSG}gahd@RzYz=3(W~TlI3#AOtQgS47_R7s_1(T2Rc( zUS_bX(dF%%0uM*?KZ^wDfy$2(iG{08f`MG9-XD|rOg0kuX@cCaSAC=wzvcqzlfcyHTMTzy_7*mC2cq#TvCsf{`ZI*>DDAzZ-@}Jkp;h#5=;)8Z$ z;@`CZvkS)#3v!JqH*d5w)|;uHlorn)vt4?2vaqB=sT6#+`JFjCDUD7k$_Nt|L^p`s zqOP{r4~Hp5yGE*8_LdPTIhs)yTc1Nt)PDFy^`c;`TS>bhf%gL!w(4=N;CGJy=b`g| z8vzjx;k=q3Fs)z=4%&;L-5IH*NE0D1>wj1rf6$-12!D5`(5`m`u$bxLj{hD(?lMUL$F1A)Y*$DWqJ|My?pER3D~SZ!={-5X(OzHq<7={J&tcTl%j`no zViRigkDLm#bM3yVa?bp%0z*OeWGH*=RhHNLF<%}*{_o$hcP?vxO`Yf@&XlS3@(Icl z@3K*%4B6pPag&0AhtSW*o+|xj&g26sM6Bsowc;SSL|5!({k>;M5gKTkdB&0fh)AP! zMR+~PXiK53{+y12pk8bNaP*@@+!X)oq!vsev*3I`(E2ZNNEn}2ajE)?#M+`^_C5}K z$nqhbmeILD5t2~^(&)FQYr_NCI-xy6Aw557`^fcD3^rf47XEqUfG;W7W&OgaP3o`y z0|AwOjRTZv2{$mu=+*>!zZOCPvmCuY4z4!YHZKyLj$0NY$Gb1h3r$#Z@m;I8!EIa5 z3*X7J6vh^uP|n`=x86|SC`o~b8?KY3hH$3dh$3e=&up9KjygPU^B9V8Tsaq@2b>_h zasD^Ddn)()po|Nk@T0}N#mrHFRbttx_NL*aEhY2p^ga-RY$K5M8Rpe%L1KD4X!yq% zJ@XecG*d5{IkzJ}GYC^Cio-(0AzH0yhD9#)EfOl{Zj0BVDmGbml{Rw3X!v?yA(7i! zj$fVQ?3vZJ_@;T$$&1t9iYkG%dXnR;z}O}1H%ngnuYF!v*0hsG)|kbwfW2^K3q!9% zW(LuL3yMHVV}HcX_XQ3LecvdG3`hG_etT$><()yeH! zgI`Kq!`#wS*vLi<>T_@DOEntKr1!;E&w(cq>1&2& znlYj;&N(!k2su(gMqOvY$bJkPdf8UCYC21j8$&a#0o<5J<&^8bUx9GQIWrV;F z6fXU!0r&f)d;{K__5@{YZsWIsP7U?G`ts7nI`Ye{54~yuLY-M4l;GPSU*kcYIz&3lGFbZPZ!jN`072xvR(a-LXAa`i2;1Vo_aqqy+C-&5{~5AIA?*cE!R$ zrDb?-lSVch`ZrlJkDO|O;wI!oQB#~oK?xSyq*cB%b(N;7wBLRWc=q!lE{(*L@Z?>e zcl<8D=93Qp6PcNJT{q({mGLnUM@U#(t%t(ONO=4YY%;pP%RUpM5qqs~EKMcOs$~lL zXoE=_ENm@oVF zbv(HBS^j$clSmUFonv#w^yGSSlzP8 z^TZqC&08A!F8CGYl`-cf_lX5&59KC0*D{cB+#HxlO_ zLNCFe@ z^yF-%YU(vzGyZYh7Q?YzpqXqr+90_T`|YDY-C_h2t?3wL zGkH%z6hu5C1*Is81 z_c0;pq!9$8dGdJ?%=%R?bWh^jqx6ZPQ5kd8nxgB|TtqM|)2h7#Qpysu=`^_O_8wa6 zzI5!K$vUTt8CvvgDWDZqV$>zDF1TgzB3T-r%C*|lurNc!%wd2+^=MAD_10Ohwt z@9l7H=Q$THVvCURChCw_S@wv&`DXi7 z-hx9@?IN}oG-(gJ&6b48iKp_3g#jXj@x5=2k1kdfeYLQnvxu)OPEkk_nZ5ou@vghb zV_`RDEr(!6Qp6o2eW6@f3)A{S^V7)scJlz3$oF(%W!1?hnwT8kNJslBpQr69L6zTyoZM%zo-kUCO=Z zbeZDdZ2qi-NSf{%_^fr|^;s(z$Ek^)*cmG~=Z7=d`Obzj6=RfbCNF7vq%5ALuB}AW z!S1G!t&fjQEX|EdG2TbK=AG}|rndKjXshuqUa#g(@qrsqc-byp3fm=6Pav;FIybxV zQd>CGGhf6kEqiBlC0?ky|1uQ^kTwkysgtmZKAjiZHgk}(2Kfj}P|qPup53_c90z%Z za0S0h3^IX;q1p%|Ek=LoQ#3a2a>;S91$^sWpFX!4S~~{|F9HXFpC#H>iI%Z4=Z?K> zH|t%!>&S~&p%m7aI!3&nW;>N64rFt9(<(pyQQ6*yv1}8|nBlAswQoF9I^cY~EVT~7 zhkC@6$72P=^Sk@gOd>HTM}M{@9|UGPhG3zBq)cq&UuY2-<5E*CUabh^c+O(ixqRCi z#enJIMv-ynHdBqa1l`L20xup1ym~!^lZ#{EPb;Y;kTJ|WpeISKXy=*q{5!ron^@Yha3cWja_8Yi`XZ*49{|xxF|)Pcc5u z-k1wRhyiE9Q}D>dg;_y=xki0(r2D2bZ5y+5ow~W^#BuEs9Tl_ zi>G3Ak{St^+J+)4w%@bMQuI8-%qO|{ZIn!(2I?RH{{H8V2;zw=D6)Srkp$42k>EB!5r-VZU(ykRWM; zFWHUjLtt8cq2d(~DZ0w|58vp{$8DimQIqSUnV{tq2MBw#A4+_f=SNKQC9(CItV=Pn z72Kr2bB{w&C^@lz41QmpL$mbIBXmE+mc#Z?JG?qqy05z`zHy8L^UKSss(MMv;;`Vs zUnj0j?QL`^;ZUEV{UNdJ*R})s$!xCy=%7-qx=Y_aoFpF0MqfQ*76Nf3SBJR7Dj)%q zZ~iH59m*CE%e?%JI%xesiq@77;bOR&bUs;&2y#-%P$9uV4 zrr0#RpKr=f$2d6;uYW47?6~RY^Y&O@E5f3M2-9sy3k8o6&}~U)W%__vQd8?OPwaCv ziLVVRb2X#;W;IG@uUk={dNbr@9EX5F@=pbaA4mCR)`yr2L~xw!@bxi~%&^V2S1;A2 z6>9msw%FWs!!Yrf3FnC^u!^m6B!K}Gv=3rcxMk{sx1=6!SdgQ`i|Bj<-a-@%?Pq|J zo<$W$HnkpulT6S64?^o%hNUNaE5AZY!MTGr_L4QCa5}0K!%Y{kzbM0g;xLFl&d}p8 zE2N8wU1PH4YYJr9{*kVeBC#|b?@5bO7(19<&AvNFl0H=#*CkZ8Owb*qGd+Me>SaW> zuRAE}`1GOE1s%QsapS+m_b_!US zW|=9(x}1bIBEqDgr?GNRt-j+doehQK)cLi5oCecY~V#-8yY? zRp%Fr2b7CtRRT1Z|Byp^9%Jjc7KS`DQIezgKiytZY2%A;?$O`JKqxe_r(1Eo7kxUm zxwIbKh(E8myY(zfg>xXKUcc6{yJmk)?By6V&653Ke&M|Lqn2&MBiK#(+Hn zwW(sti@VEvjtoFU00a4LS2p=?S~Vv0-$R{pI)w<0_`d3;Y+LDxWfgEH)0mw|KnyFX zOc;w5+pe_*RncHLqy^G+dTT{ zNpmA6{F@6`q@57Vlr{68%G~>G%Z>m2eOCV_rc^I3RMZOF3!~q8z;K5)&$JB+$0oVDlP+9=JXIKF)H0+8T&Ez<*yteM^W;wns(VOptqh3UYzWSQj zgP|f!<2BBOtZv-|*GmR^qau)wp{IO}W)2!AC4{!Rs<-FiX6S5H0k2a1O(?ptnmbLw zg8kj(RdV(V0g;S}(ILm=5j3t|&c0^40r)g6m8c2!^nzd_X9^TnGU$|{wa3^Mb%~)u zyB=|h`g*OO;O#O_A=V~7UU4EAk5I6%#}#-5iIQDzo)I~IW9n9=j%07*Mo(Ca1o*P^ zT7N-;{lEjJOE7;J_)3_ORVk;AE) z^%03d5{W@A9iiy{Ne_9kO>(3Vww?Y7sOAwHL;#}&y=Ophvy&;13sRBMlrzU2l{3R4 z-Y=y(aHcENYf$W9F`KcM2$0Q;4e?CUMC+Fc%cjvC5T@)A5Fk85liRgg;|vzW2%}Kb z3@GKujY`yXi78`7L+u?9F(`@1(?La#LqP9GL41Fn)IITalXWVNF;21-*i%V4lYiOG z53tR=qsAmKRM2Hxk3X!U&s+ZSfv>2y=NdJ>KLz@tsx}pS+(rNhq*ys~Rn%@~qFM5W zHzUB{mtpaa_|I2Clra9Vc4C(6m zJm2_*MZ`W@X1YfWX(|i1VH4z(k6%NmA<$ILH8(I0)RgmBq~9DUl=iiu-wPxBNQ+27 z14@7A-}1q1!m|2EcwqJZ!j^2~T7_)==o`TLjsEVOW=;>#j*DgJ)n)}_&x?v2;4YY>TK9o^{T z^ap`d?kIO$;HNhVc=qU^zW4ND-JnGOu+NvLrz6Dg-|MKD_LE9Q@BvIQ8L6wi^5%)K z`vs*sqRU7MUSv$o+~Ie!`d1rUEZ|el|3%$fhQ+mP-NH!Y?iL(^26uNz(BKY1f(7@W z4Z)q@?k>UILI~~_90HBI1cz_2kL>fF_kREGpZh#3p_^_>W>w8qYmPByF~hu21~is0 z4h_`jI+w}PpaL+cIS1fJ3E`?WKqf!EI#r6hs7T-gVyH+B36C#or!4=3zB9t%$l1i>*?9y|g}UW|+_5k&|AFBLt>HCY zLTlct7;2c5h)4@Rm0}w%Vvn@Pr)Tz)cO*nIxJz!McUEG(jreTXd$^PUv|oc8=ddSl zBP=6(#H+)DNM9`HUU%~A-7})dP`8l9t3WPWL?NV}5>XBlidq37n;d2n@`Ny3;1BE3 zWlMG;D*BUHV2Xw{@6qOUG_kOp2sTuTL|R;fYKmvKr6QlJ<^I0p@P2|b5?5JFQL|C8 zV`sBtz7kiNXh-HiX}49%`)L1U*;;q)8p5c4o(jR(217QjNu&VrMjM`PszL-#iX8ND zKew<<0-0bNnTNi-rRRf-0|Tt2$6QzB?QrtO<@KK5p~gR6Eyqgtt`w_WS;sB+je*}k zj=hoH-yKGFhV*v?5d4fMeXcLXTMteM0KXv(mhS;e5gd=7eB~F zY;keUQT4GzUo~Dg_U?(w^@KjtHB_R{Z|C?(wDOx7IoiGs7fYX$sYY{M@%t1dn)A(O z7;A!`+NQFPSt3m_gNcdPK-cn+666<5NnG=ls+84(Drq`w88wsabJF>;6~o!`mA_(H zr)Ye`axb0x%^JT)$ZU0T32JyjX04!$Njd33bZ=Y53rQfe-epJhinOC|wRbvcwS2yH z^hk4`GhQHBfPP2|aA4;oKlU}U#HejY+LK!9raDV{ev`$EjuM)p5smrBQE}aHQ-ZHb z$DFBRtj67NMjWs_vZ`#y7$$Q*H@VwzDh0c?yvMhCdY?RI(_C~4j#pJ165NOF%Xu%U zp*psUwraiTuB~&1J@MnCYRNM_jlXvjnw)ULl zr#3dv1APQ?+MuGP=yY|p-LYvohv*2Al! ze0KZm*|%DhP}}$AT{fgF3bg*%$BD2yBwJC1?m8l$E0mS2EBCGpXLC~JnQ33^pUc{i zdDKT>nxUq4?qii-EUa`gZ+iJnb7=hfi3!Au|19DYFi+z;$fG0}|HQTTy{O3W5um!d z!lo6k>9V?%PkU}IWnO)9Pp#W)!Y(sxLw^&y)ZdYBm(axDH? zd;N;YjMRYuaoS5lh{W&dGJ|6rd>QtOAehMEX(adzSpW-RAVTMp9=7Ti(@bFCDK9WuV74>i`cm*d5DhqSJA3$ zg`Av{8wywIMO}OR$r0g6jG6r@@oo15Y{%26!vDbR+0@ug@+jI?1it?2y8c0XQL*#g zz7c(GHB0Mg_Qx4p2cocLjz3-i_FtsX=6Z8D?_1W1;CpPJ&U^GMx2kiSqi}d#lCu_Z zjRAt!sn_#SE!KW=zaA53F0vXv*63?1U1r-qX=Hs`|D2ufT+?{J7FyDIY!J>vNW@1$ z$lzx`>(0#0p-Stu_Lh+S86XPGDe=Cwwz=QLI35R<`YEds*KWTxD&farqgNN)tsxC~2u8y$1Sif+u;<1`GCq-BSI_hei3 zKjg-5R@Z8SnWRbRw8+~ny>GY16Q-)H{OW$0pfM)^`78CXM=+S)>nO4>1jNX(V%QR4uEC3TkMzQ>6)5=Tb5h8&l%y4xQn!tI)8 z({+Yrp^bqi0$b1#3=#riazYRB6==NUJH{+M&zDRkA@yCD)fh9$t( zy!|msa3CeiJ(->)oPR&cMyB7Y!p^Av$i5LnpxBxBS-c}=^&9$13Yq&NJx_ZarUH%{ z7F!7E#o745MQlxX!+OR5)O#|bG&mr?BV1DKz@y+$~;Hll|2)Q``mZ>Ku(g>o}0X0Llli)vo#Qd_UFI3t2>8HOYcUbh#_~6ZGMW%(J!0bN``mz)X@e@1x-aZxCHR zPP<^uk)p;f`w?Pru-tycW}FC)%>G_^r_Ijs{D(5)e%5*q7LVt3gP?<8E!d*aT)(}$ z8u`ZO-bh-HXAejw9!6?1A?oXALx(!EtYxa|H?1pe@?j^C5Bk5m3AIZW$^20D*~+!u z5hLe=-NpKcE7*S^5(V(~e7>8qnjeXPfBaG5viqqwE&M0Uu^?hNsHX&7rY9>Cl70{t zU3VbVTevq`c%72|HJMPLI;CxkIWcrm#6Y$=0PW^>UYAv z$l@;vM{p19l(uZods*JHk4#s}1xYp|f}H?Yr>MEj^o+wpekB8E7VoC4?ulOI3k+Ov znlbZfd|s>=JQ7@g+JbS)X(-$CW6yF+ z6x^6)xkCpc7}pYn_!C9zO>2+s7_)L7(XK2TIsxao^j(*?1fR@4T6Y*}--lcntCP%6 zYpgu33`ni(Un=uHEDOA+Zu9F?)o`1@iVP}ir{*J*=_lAn!~#KYQlwCQtt5lpb@`5i zfGku*X?*!JYQj|1gK0*~eOeCgrx0Q!Q610I5OAQ>T#h@N$y{}^*-~pgv3d5Xwug~m zg@feLv~PGNiY6-|66I;MhjZz;<)rQj#?WEKw4VcuLM1o{?YrOU^LII;f}2sI>|K%S zFCA{~=m}sfnrtuTBzj;8O^QJck)uA(*;#g>!noUCLRXVLS#aW-Fai%2lslKLT1&&Z zji~l%(o5NOCwple4XNlLeu$$>Om*q7C%K0H&f=e|vicK&_U<8>`;bOK{cySF9n%8x zN5xAGN;_kaE$i~~iqMfSDGNJf(M{kHfq{PGI2sxQbz6+hA7omggrvK1{>CB#cRrbVs7?2Qu4prY$n#Ox90k>_y^_WmSsjDd=(TuRqU)jVJd1Xq z!oX$*o2-=wqv76PekzgP8v|aE%k-LcT+l9|l;5_2!_ux;8g%?fY(pX#1^o3~&H8Lo z1Mg?Fs5Np zVNuj>NulWDOA)z=k85*|R=)#kzLujQ z4-?u389$LHh(u2C%ufcG7O(VftdhQ4{^g>1EOJoMowL5ovchiJMS+H+HnJNsM*N2; z?_iQl+*tjG_n;Vf_yOP)HmsGm@kHXo;Y12a8e<~9wxdB-Lty(Hvo5BTk!#Y`7>I|b zPY$|Zpgbm97iQoA`ePtJlYN|DtSEjS4aZLvyB3C?O0E^vh^$<6#xQ}rxHbd6F$6lE`v*hL*<;3&QfsDTDbY4RpG_V^4n)k< z%A`(6P2km|)QiS6)B^XJf~Zes`;%z3IKfBzy5a8`n+5V59Z2Py|Cm{dp>s)mQ~3PG z`5WE8)z&}_6l=x! zYHRuwOWI+gb}ZQX-`5wQ0$~y;2!-!#gxSlQ88-7(Ekb`dWx6cV`%fx>?D@-va4PbR zqbSshk!xoY_tOQC(=4Kd0=7RO!&dCYis6sx>XmWmV_kzL;(AYD0@j5Qrvi<|2x)&> zqC`4mx}a12SOVAJeRy(F8z-gTPvL#49L7}+67GQ%LM0s%YSqXm^?m0 zAg`Bh?#=Jr6p++TE#RNTS|EiK; z%~>FmB;eL{X6#rb9Rb6X#5dXQ+5h*jzKaazh*d~;I#K|-)p2S5A5wUDebbYfLHv5` z9{Ao_CRWkm=!i>SlX#Oc(M?43;&=UrtO~mwJ3GFo68}9im_!-aq8X$=9Bn|C-VLcnH;18 zf8$~%ky@o>Qk-{hS5T(&1sqRDsgraT{w}caHc^pF>Dw+`Q$?^SuRw~*6D(Z*twVZ9 z`S@`!$|#$^=PEef*Cnx1V#V0%u$blDZCT2(CP5WVl*@i1zvNN&u?aAttYeX5d$_(VA<=d@w*e)y~)#5>JUYq`pYjJQO;w zPQ-n6;~~JxREX>b&Q;?gZCEWn@X0@8llxu{^>Y{nLC9{XgS0xIE~R&O>m8a~2oJ~^ zh0;=RI2gY6v4O}xkx;YT}bz_QW zo_?Z{^GTgJiJO+iEpOXL(Qu09DC-^t+}Era)Dc2J{nd#_sba*J9a>!3`YV;#jY1^e z^RN3IpnS{dMv~pHXqO`vF$KDkZQebLubWfzNf#2%a+B%?&`GB+$ZP^P6n7IxHq+l2 zleVWKqg=SMZ3%nrbeDg7?CjY_3KEak!e8Czef`@*p)w#Bj~e2kOz{(I6^%{E3k~f@ujSd`{3(%CQYEdlc^R6B-HE6}5GAp35<-`W;d;zv!-9^~`Ni=sQ&D>XstJh5BdHI@MqSO5Ck;WMXgnD|_Y^ zwVjkT3+R%7MDQ4Q0L_PlLw#2SS*X{MJiQ#m^aospxXhV>0!Fs*2q0bY&sf^@AJ~I~ zurK&l{6KwWN-<56|LAnivxi23aDSfwg_vm`}`@ zzS@|J1vlc202=g#4<(qTX_X`IN>Fu9r7R0d$Ru{m!ucnvr&SNyQwr{BT93d(@Z`tt zoZxPeQyQ>Vlom|#3eQM`WEthpE;e!Y zB|iF@&~w$zIH^*01POE?oubRacy5XGP;}z$NNNg9 z8Se>h{T%2FOitK$!U;YP+Z%9Sc#!ht-l(f zBM>KUtB=FlDLI&<*Pv}l^cVj!kFRDl)Kc%zS;TDqiky|k%=0vwXoDN>W5Rpk&axwK z3)^ZZE;P0&IM#1gUMYEB{Id~L;5RMS?FAh8zZr&hZ4AjMQ2bB}c<;U!O;9;1R6PHv zDH6>z;Vj9PO_l|HdLy2VnPtQ||NQ+oe$+&FTA+ZT`4uY93Q|wfq>`b~O)-!1E0$B0 zfXuT6r5r!w5*de$8f!7JF2svvUh*4l(};&fTwnTqA2rt6U&>eev-5jl7zCEdm2s>ck`2Zpe%eBP6{n{M0&E6}qMQ;ST3HxDY6c1cYAZC%C%XAzx}Iz@dh zRvk00GqllhKVdXs@NKS|gn#rrc;!(gmsEbf5VhrPmknwNPIyJ-tatMkMnei}PPkyb z6df!-X@DcrK!5JQ5HJ;STS7?br;)F6%wMa0!QT|uMmJUqkDs|rnOWTwXXlcDrEh)l zq`Vb<;XRozYIsdu{#vF9xxVi9JRtp@ItkXM>{WAbUi|kymkJ)GU8%0d8>YD$S3y&< zX<-45!Z%fLXyW}iEy<2bpDyMQ`i(CU(3<7LzPJ)#+?DFkU++2^qR>+1hf)idfvbMxo<=JhO(H$9x4Jp`G2c9WmJ4pIXLQ8Vx$R{UJfA#%8*x=pOij75j>t<%2pXpJME> zRiGL^ONtF2R4$lcUk^@M3maPrYljV3BP;q&LBIXpRqu;9=DRj66?c6UTx)wwVS2Tc zM%BAk?E2}2nS|deo?#u3kUR81lEoac>p_)!KudK1`BU;>p_q|>-cX@OYh0%7PO4Mz zg825*Yq-~n5 zv1y@aItvLiiGvEr&aFe`y)MId;fh0tjf$zBW)#S~tc|8}ZIH|98AB*1U(PfZ?t9+k zzqf6YBfh{Rj%&6_uleXIEr}ZRs%k6c`pbI;IyUVBn=SW|atW6|NrNkI=t%@_9U#aF ziD`|y6)e1|HmytW8CLXDJj+umP2Y@{n|xfF(pStg(F-D`M%QbOhDq3(FI`@IOplj zGf&s+~s>*zQ!$@0Pifv(Uddb;ITh zvXqd|wV+!LI(>t?zlXVlTiRaAKkOQ_DQY+WQm(aN5%npib_{GW&~huQNp+1Z3pFm* zKb@t$@n^FU$bFWn)C)diQ}4>9(mJ&HDo;){a$j?4xH12QBkMOk{d6IowUMVK>$_{^ z@IXRY`D}7?fj&~bjy3sX1Lo78$gc~>kCvERbb8)}Rdn)U!x}G2V=&HjT7Syl$NVOc z-@J($H{T+31#Z%`r9UpAmR%ioE^f9DLVD5fbp6)akdSy4DnmXeGoSjy9fUGCIcCGx z6DlO|eXje<68iv8Bw>#$?oU|yt=%u5Wh)#I_3JR{ZIT(UR65yHy zm&yg+B=qcKHKPZ|opT?9X%h^C1E7MxwZc2~UVSg*OR1rIg`%y?j;uQy5L)P-bu#=( zW!AhG>ioM!a*~P4>`v(cq(|daGm~72L9x3y_F&oXh)=aF8mc{HJyWWb7$#-R;p$XW zL-7MM?KUEXPjNatQDvg6{#VD2E<3{v#AbOUC8(-%gPDVi-Ud;`PDxT7FW+qY;f0kh zeI_kQNQdUP{9x+0*c?0 zl{XIGSdMPierClRw}u@FcCoi5TqY9nnh1c?(52oV1oIe02ts+t|a?5ZroGBXDeDq4G+bg{Z?kQtDG&>`*)y7D7YUc>>SMHSN#IGFM8~$7(g+ z*c8y%p@0<)-8THmt=faWl$7Eysd|AvS{pB$6H_Ehz)Z?P!3wC=cSyLEEaEbrG`tlZ z8;e>ul^d;rbkjMyLE4wKX=3NBBlXKCW8%`vUxT23iWuI1fjcP=yEuCVY&B_A^gx&c z_@1S(7a!&|eLPloDZja$ET!R}^Ah)N%U!C6bBB4sAS(E5boalJbG7ZwIX7H#rx+WH zQ8yc)4l6kV8m|fSD5%e(1~0rZUTE<3N&%qk)gn`1d`j9rt_t66=GO7ciU0w&3Ba-h z6rA-+d!?!3udVmT%ek&J>WteQNs+YQanhgZp?fnTq9bg&8HdZ8P=WVp{80%R(yU54 zK8i`mXbuBKcbQuON3b?k6 z3m%O=4~3BX(zCXo2Fz5G3uSw>pg^EsMgu^%6`_K&@3@{4?uqD`C(PQ@Xt|Q^uppfS zd_w+1UawX!EnBy?3g-4Dcp1{laGhOmZ$w4E9vY2Sd#) zqnY4^Fz{d33^6n?XM(fnUIVcFe_h~#7a~I>&NbD4eeir#tBI_4%We*C+dzh7Vy z0fojn{x7;V{pUxaiZ4fHReV~grR8B7 z*ZpTrVsMBrYUx(DBV91HryLg!AYmMJ4{@db%Pb{aU~Zc^Z6Yfxt3;LvxPRJLj2Y8% zzA7`4OjrQFY)Bv}5NNzp_wTBISfu7{j&in$)AQqX|3`?9&EE}EfyuZS>;GSN0ZfsX z0hB=BC0;@H-$Mx0sN%W9`ELvV$BXw$z=TOyqtFBh4F9XplJ*Ot{~q#}k)@{t24Tn^ zvm6L^{6|^;c&)yAi~3(v=;f_g!i!NLyrHu%Mgd#Wsc37{D+`@{8Oah}D!V(f_tQ&a z*73LJSBHY)zkDktuC9783#ioUgl^UY;}a8UnY5}BWXXNhfdgf%I}9uSV;I%H038ok z*<}Am1AJkw4(5|5^JV4j?5Zl|<>X>N^akHH&CeyTc6iBOY>yT;H*23?T$m5UQcg8H zZZ4LgkIVnvkwhwB+N$1ou>He|hze<3gSh72T@UBOwCwCyMm7djt^17SlD)xbkSTfA z%B#C?fB!L!%6k}qX!9<*{~z=028j>S?0PWA;rrn6e7E?-;j*i;H<=%zkij=;uU~Q` zaQ5)G1y!(MCfT`El_Zoc> z)qo! z$Td8dkycK4Z^o(l@_Gys5T7;Pq-eEoJ%*QwMEiAFAah5?Z2@)k%t=^km)4viV=>(g8` zJ?P-u`Wc3YqvTb%RVcM%)2pBKAEi1UKAZSF8wgy^PcatiRc?fmk^AdPcK_v5#4a&Y z8aA}f&syn9nq|k|d{BfzG^RQ1iU3$W*M6AaXnhIT8_e&@WFdf}S=BLSt)JpM6Kj$p z5%~~6@nd59qv8z$>Vx<(buyqd3QP87Vo1+m^Er5TJb3je)}HK!>y}gh?ZU*~!T?%X z*`#Cdnyq&%abmdG-6g*Oh5Ogb{PO&7Zrpmb1ytVdeUh~zk1J`y?Bn>;AY^3ckzaNE zrZNgfvKks>!8Xl$0n2$CY@8HDv_B-eRcCtd=ml`Cq)M>l~^1r z#*8h7@Y>K^^3E$#8LOJN3%Q!)jM_=rT;oGnjheb|&)bDKq9}%UCLxqIx?Uij+3)u; zLtaNK_F1VfzAdtQuw7wrD9XO+tq5|BJjLU$vR{?nv^Jeb>lyEox_jS>F^esYNoM!c zhQsO{5aqD2oOrJ^?_TC}KR*H6KHe-+LpY#ks5sr;?#M_#7{MOCzA+z9qGy+WewHNy z=P$J7RmgCwWqR0&S_iX@t7d|fwnfVs05C@^C?sSE_j97{eu`FvZpzw}t11g4bylFN5^g73v7nr;59&)x9r^2Id=I z#N@sP<5B)!@WlVB;9q{8@x&Ycf8Pn@D29+|WIP|Wh5x=2mDGWBU%My9t(?UE8AW*4 z@kL5}bscUgE76pHGC7!~#@DpPW8!P=#e}y`$4(Su8y%9WGT}q@kV7rSi)&Kwr#^AL z**>58cWrseV%Bxy=Y8ze1M6e+CBIk2h-Z|O#51dtilS8WYp0^H)8G9wvyX;m*BpZ| zWb8sT4j@5j=%g}-zKrocBXo58{H);c!M`^oq1G^8??f5jqNNSd9r#iK#8PWlMY~_R zwHFw-2P;Ky?XBXLrDcC#5A|%4*!o=jG$<%(77LGvC@`pdig)}J_2N8#e11FCXeG+; z?bNn~XSUF(Na7m%`t8x>FEmw^gktl;x9P7tw#*ERfb27MPH8;Wtb8YvhXzx*eJTv} zBVUDO@s~O)RX-0rnU+^7F}$@kU!3xSObpsS-4$|u%6BECC`PVpXm7YnrH&%zQiGL3 z;14)Y!$^7v_}Vz?7rP^0oYY(RHs5Wk$cmglD?QhB?V~x(oDEIse$>P_-*AKLuy)k( z_qkocCY#az?viD5ye%J?gy+#KH_?~21dkJl-UE4vY+XfNOd{$vHAjhE6IMb97`iz5 z2Lo3K+|hmiTClD!TOb279hSfM4qqssXj@GqJ`_m__K~&$kEZXoA-!3Q>{K1p&YkTI zha&u4Cz4YgqaJVNO6aZ*xMTD3U{brTtOnH~Ph}f^@qJ51vHa;-wVXz|=3>x~2Nhu) zEhkOp03GOwSlrXcb<&oj>|V?DF)AJqFXNs7PK_22z+Kw<#rdtdYl$!?qu#bKHeH$b z%vqSNCW#LHGt)?&fH4rrLNHsb^(xeO=icJ7TiwuZs%z-X1PkwK0XX`+%{@<$uQ9K7w87~3TKOn~ob;lV%vYmiVNb7T>@EWdr!gUwk8V6h#z~TYm zUB>5I_I+X7g~u}b=f&#guFfqboz4T%+Uo&qPut0(Cm(iQ(VAyd>rZ^lC)K_R+je&y z#%5$g?Oz}J=Z0*6J61a9M{Qx%bTpUrL!2xqK71I{-ivHSZhL->+z;14It0O zH^XUP{_62So-}*Gpi^7?LGopTG%E${G`Yd%HGgk-FvG=Q(VpP>Q{5?cSq${DO%HPA z?1lN{MV2o88kbuV?K?dUdmKrjpUXc9^6BNonttJH5tj!TI@}t0OC@=JRgCFcOYFmL zoisq6rTUXwfdEu3SNi5yfDBNh{E!MLJaku?7s~U8>CVv?KaWU1>&sp#TMBPzo`5b^ za}TQ)1Qq9w?z5SfpLm@;PbN%4KZ9;$sy3jZ;5=uu<`2I`@5htV>+tV2?n@nuzwv9D zMY#ePlIcL69J+;A5res7pCy^=KvHsTvZTNotF#2`Rpei*6zmIxxcIj+L-&o>?EX7F zc}L?!ftUGAkmfmCpw!Bz5<30+;Z|1q?7z6-v(kWBT;t4J{=+N*Vw%T>OYw7`KTQZ{ zIGt=KlM9pjbk;ELf}Mosi=pPrlP!l{CrsrUbW%JD8nOVKz#0C1y~q4H2h$Keui*r{ zT7Wua>kj!a)~Xy187*{Q(sys3_Pfon5^xe3IQp24ScL$#Y558yC>S|<^?tA8H?GS} zHS?K)dq8yKeHGGuzP{Wh{F}o%AwPZoQ{--3Si~yw=ZGtELSs8GEB?*Z7~2#WD28#L z8rYkV7r$E;E4EN+iI){A{f}+$4;E;bI|TPu&Aa++lg#xOb|et~pEp*g8R zwd{(}#v1o(+~;>PR8c#HrgTol!hKO+dgx^)wnw{rOW72Qrk1Xpy}@mV35f69kljuw z{Kay=qq}cJ*5WVjL@cS5rHLWL?}T8?HKfam0X{ALwPKnDOC2{f5KVf1os4$t8l*g{ zxnki}N>Z!UgQ%gcm07$j3+qP-MHB>j1ubQ6)V@jH=+Rq30Ec`T1WT5z_w#AK(_Rzq z6^HBTxb5xl9_7VaJ$n?&r-}*w{aDiluMr5@+I(?W2uAv~aMd*|G5xVDkj$jXg~gpy zkNjq5&Uv-oxh=EIMbjW4)pC8uT0k8wWQLni*K?(jA&=Q0bH&v#3(8))xb z#-&5!b3OlfBuYw%@9jkKC-lc0rFl7}r}qn$7ss%0$m0e$A8%mS;OyLoRStQJy6m0j zs|D}*iB=wMGm^N};Ea+$b{vtOMR zg;U+-reZ2Doh6V<@AgGdV4-BHF>{GYiAS=fk&7P!J^^X)2D?N%DXwr;`AOC0KWzQ{ zQ&^~Lk#FI!*4I)yGukMff!CO2@NQY@fi%WJ#nrGmCHT zRSivsc<&-|n4ZRAbG;GtVcKLv-`TZ_<0og_BFZrEgjJC>C@|G5ADH-|1E(3z{gY$T ztC<6GAg>)U{h@if&0*t3ogJ1K=<>@K5P#c&E&dUyR!GZgsDGz?c4gz;y}!f~5DJ$^ z;q@CfGK*mBA80tnBZO%W2;Du6?9q22SRr-Zw5WOxiadC9eRCnNYIHGVVOGg&&U5M^ z1@1?5Ui(9kP9l`$K2m*+ydBcjc23!1IiYN3T+GzqD?)IYO5#gNnvPLpV9@e39uR-^ zLy2F2s?(!??MQ&`(n~vV$;YJhg4kNgbEJuN%5PyYnBp6$?=)C7CvNY(2-S?_a?7&s zs<{U;ZP~m;_6*8Z)7sj8yU1fEM~t)ihV$kVhQuntiSL79+DnSL3?WZd-3Duo&!(>` zZ^ysuJ?PLnEO-=i&AV&SoZyBq{AzpitBuf#ie30eUwQX^O=@Q#y9SMP6P!GMm@}v- zi1}b;7U12EZcoewSp2c(UHEO*+}J;xnhl(`C{D7(DWa;O_F5j$laz?Vh~6)Z(+CRc zWqD8WGO7g@ZvyMt5w;d3(51^JAcGFE^ERy0d)02%lfTljOoiLA#pqgx&1)?MY80^v2lcist0~ z?>D{oJ|2s2h&!I&pOgt)KThOIo}cz&4BwnB1{=Di)>}=>%pIn_Ca-!F&up7BCIFof z|A_VYynRcXaNT+58*!Nl6Xa3*(FIZA_U10WdN6b6zTI`X<&5R0SE0@(dfMOyww{mX z;^_Y9c}hpy&p+*SmP3PmOW94=t@0@Re#6<4oo(plHD`BmNYTX%w8W$TC58yzVXL0o3 zVX?QyMN?3ZVo&LP;DYL@`)rABPQO3q<30GtZx``aHGLd~iDP^B$>zy5-pGT4rl3I} z9&ug^^qY9%jH7a#*Q~Ef%l0|r!IZ;hleIz3MSI*c$<+=s$ZIz18F_mo`tS)){_0D; zGN7HB)(D@r=SLftoU*ROK#$x>U;ilz&HF zeP83#iXzcn^|XydzZt(GLQG?E?K9CGL7=B%BIzL_f=2Xs?IxWBKGeD zR~==?80TdOIP`~LWS?)3F!LYGgqm?>^~p?BGR{zt42kDjmoIa#>AmFHqy?r33FD4; ztiiah0ndrC;j76i#UldotiaLFlMEo~Z7ly9?g#f_pk{%l)i$GMS344sjM>Ym(KXhoF2YA)!ujC~&8 z?#zKSIHE;ub>nH@83&)V?=%rD2*1)hshzx}8;aa!xla2)RDI;u9P*7n4)_m$UBW%x{=kS>?#&Xl^p2O7+y{FR< z(Sey#HR;8ftyIsdRt4knB%Jf@NYS}P+RsrshMSr+!Ec&k?k|%d2J_!?S%oQCYQ+QL z=j^zYqJIdf+h~_Te#pDGm&jOW^CCr#vVo-Ul)xW|u@6`NlqyAlJ=XdfsYb|?HL={jS2CTM#)iTmXQa4kqJRVyhcgtqRgR(7U=;|Tsmy{^O?aYma0{*J57K}f1C^(H z*X9}htU$P3WY^V2-Ri(!KZdx53 zo%0DvjKKs3wUL|Ig10!%>D8qdvcsOnVk|LD;n$g}lkR#n^T$(+zgL1N;&UM*ZNsxi z51ngso!#&eSb?7i{-=@XrH$vS4C2$$7~j6Nz+F%kl)go#A3M12FdgzDiHbi+a0lk9 zv-k&}5FnivKXIHPY%L{+7fnD*T@@^ro!?7^!2Dawg!O}`!rvt}e4+t$E8KQ-)X&Hr6= zO*poO6G~zCo^&C*L1qWax4`a;7mRyd%Y}pNN%h4_?l;?^n{P8B+TeO^hq&yD+tQ^I z66QhT4^8>4#a*?hn&~I$p5ApXb9a&NsU?AoBr?5526?ZV*^{n`-{Go^!FeEFYO=z+ z)MK`z^EYA@pkVc&MEE#U59`plSDU~P7$U7p}*Fm)|)c?h+3C@e)M5JAAt!HN-0j#XO9X~1D zxy?{{ZZt_?nFFS+bNZumdunKtlA?MWpd%p%KoOF88}Qhnq%dCB$B77J9u#hjull)1 z2M4i0M_`oW%f0DQSBHU=nG}lnV(4$JO{Df#E5-)dZNR}$n-fCiUdCm`E$PoCShihE z1Q<`Hj6Fp0xvZBn+puudSFznCGlZ_e64A?NH-18N!X@X?wk5Cus~>y?nICM^l2|zP zb(ulW62>h_#5We|*MbV-zOH9Sh5H(F+`pi+Y#iseo`o{;hY5m~c3NXh1Ej}f9M}ml zF|{$&pn<@mmSi8=N2s0w_#jYTA{46ffVki84^5rgP4YAu$DoXr$KfQUqpz-_HO4wD z+fwz0um{pG+~VCGBtrtpOVu;Qooe4ZmezalZdhdYf7Z|Cb9Wy=K6-}$!~&V43gjwp zt?X{|rd1z!FLzSDPD8cQkGj_U1qG50x(|MCf8Zc##Oq@{KQtCj^`RE$w8Rux;Hy|bOlj}1CH2nT(jj-70 zH>HODH$4*N(%%6Fetq*nzQE49E!3TT)ithzm%18nN(p_>yW`WG2cH}oFOax5iT?$= zpdAoVn-41ey?;l#eHAChjVV@!_fY%;9z1X8rRa9}>%%**lhuyOseubJCK!Ta<8Z-! z@|1m~Oi*#NY1_46QsCg=!-#SDTY)Oyyd&2=r@ zCiYv~!6Z}xg_YC4VE&>Iq@)2_tbpZd#jlI$vM3zluO>%uSw}kR)n@}CCB8sJ5vu{C z5P1_BJ5&!Qcg+@jVnPwH(Jo*nmwdpT{1E7r^)a`ntvnY#R2s8QID|l}e^0vzJ7-CV z9QLmpjW|rQfm)4hBu+Ffg`G{44`l3iVo@r2!_PFn+bp#4QB*4JTj zeJb}y$`6{U8`;9UsSetTPl_#$=<$k~2sg}#wJiK)V;obPO4qX7Gr0ASX zE@GGl^AWV3*(ma8a8Lf&P)WjF000@s)YSnmp@)B5d1+#<+a=NaC>D^HU*v?p2->M) znS=lSid+sCbg}@We)Rs7tRvcA0b)P{!gvCfYF7A9=#v%{ZGMsjBrrnBD-9WbAB7u@ z`&T>^dk8LmnwAQFZ+s}J`h@E6faI%9j}(w zMm+q_0&0;QwmLq^EC>EaqbLxeL1`$cJlES(ZI51d{HLXiN6rO~f%uJf3suQxz(0)- z_GQSvZizdYf3j1I{^}h%HW21UuxDfq`@5vPASf!;?2++Hbvl=5>v5%rcYk#XQ_SD) z0$QwD?mxQtFR*)31_;{KuVBsn75)0RE&uTX2CyIo)OYE>W5NHzRi%L* zy>obcjDIk!rb-MBjX>2Dc1T=rt@SqNl^`8I<1W6u42nv;*kKpq+s&(tCX!!D8I zJiNiKI$c)%ad@jr!-!r!QXb}w+SKp!)+O#E*03v*en`Vbo@uripbMS9kp^arPMjf!?X;|!daW`=lE2>}zYuT-2MQWc1NjKb&R#e{ zn**K_FD9o~q$m|vOXWBXn<8lr2IVC{CCqvFef;1cSUlhH{&R?Sr8dO`TTPgV9AAD$9miu6}*xrJ$^S zQ7T*Zyzdd*Dn-FiQ>yPeM^0n7v-kM~a$40S*Tt75ux{RS0-=tjxZ0}C?TRaMe04dA zMKb${PL$;q@nR6EH{g#^0z8vFAaU z8R*W-8;s*v*XuD7hCN6+ZP&I7PR7AzS-+_GZ$ko41{!EoMnTvI-0^@c*z!{{zy#>U z1HiL&YZ23hJ~dC)bgb)_6$I9$Qvrbcpgy<;LkvxdCqw`QBlIIM7-SLw;v^{)GRtaN-3ucm|Kt?it2VIOOwGs6p(Ij3> zSaC4uIpG{ju0o3*V2Xk45RQN?OkgK2m_Q6HuKNi-}akUMev zT4*TjW-$Ew=Mm5-*JLQY?p7~*PxF7a5c3Me5nY)WGyf>XQ9CE)W1!QMMysNdLfuka zyLCEfTl=EN=4dHb*nduOvEi)x$n?GLueF+hnX}}bu#BkruUpsesP3Xi@$_eaHY2;iaP#jR0dS=IFkE@m~xx1?*qWW~uE>`yUu`yu1}w zyB5n`YDN}sm~^s5I{uV(bH030?9B^|E5Gik`!B<2y>N&)fD;c?)%7u}5j|ECX->}; z`)LJFRA}WlHeR}isRvlRf9}!5_Qm7ARKAT=vHIBhYFshq_Fx@uh77DGK<>NoXUDo4 zMu_ueWz4|5Z~~z+i~qmst~08srCkFDR76lfK@OrwM#U(g3VWfT5}v-Gw{fn*L=1lYe} zBrMqD@W6lU;zJm(%AMn*uU>9BFqF@zUQ7xkm$};FlXrXQf@+YY!V|`um~Njgof&1Bk&ba!*X};AgVHJpW$`CU}rq|54!~|9|I8fORWP1pMIkAdDy0y)xOE3j6Gp z9bHn|RcGf&=bKeczw<8Vlw1-A(}gamf&_<4RQh!i1J(Aywy#J^D}EwCtm5bw#4Y&M~grg!UY-dLwV368E~j-Tge1jb3w* zjnNwwb6IoOwGk};BX8m=Qn)UCZo^O1S0B<$j@oVk_8@X zng&`6kA!$54j&OT0&78mVMp$qZry0ii%@f#ugbcZ=5cXaY1bYjFhtP^#26&HQ(n_S z%0%Vtm5!a92@#GSWb>sAW9$zD$&=48MJKM7-i!MPvZGAGgQlCSY4iFqKmD_o?L9CL zBlgeL23<1%SsUd#ZEc2qD94!3YIPSaet4}X-EskWZV)gr{N#ci-l}qEf=8tUq+DQm zp4h~VfG)~jS6uk1!Ym$)FOaKUdxVi-rcz>ZfLAl$?ZK?EiQn@{C+TnwY_Y-8$xwM( zBI@Q~UC%Wf)O9a}8P&n)>ASYX?*YHUVY~|4(Pr|nYi3So$FdyxWuXoJYVq zy_36Tz-lSIPkKe)v34><&gYL3Im;O5iv7WNWH*p|Be%B38mYtO?V7P)Tk50Wh4KS| zRq}NlSM|PG7nvpWn-yAN-Ie+8o3+VzNI4F)9di02ur=&@R&gms-ZZ_+yx0G&`)>*| z8VZMa54AlLt50|S zH1yB0}W9*OyJ3BL>ysTm4e$2`W#Mh`K|8F_RX`#%tz5$E!GL!i~r4ROK}~ zk(+Ub)ZzCpOCi4WdWQ|LyLssgX}vcav_iN$j5MpHQ}~JKeYBid99B6!>bwLWXCv4ks?<@G-X} z!k|cmiI%umGPJTqg+$A>9e}0Ie5?p1q;ig*#TqA5+w-akowdGAHv6$IH*^C=sSV-P zExl&0)Kn-&rg30%5HdN|`S|{Wb@SDPjo})tUetVYG2sJY=oTL3b5@lnF?ao)*#ifk z#$Jj>jeRJt>GTQP%@K>ll9i!=EdyD4ZsV|42+o-!;t}W5elsO!Ke&Q-Sk&aj2U$LJ z-B-~=d7d$ZZ0o@_%-l;JLQQ6-`g@7_-*9l5g(_}9)%fU^t`KT`-ag!t z?>L$XAH@w0O0Kdl)ikR#t>5m*SPi;jZGRg&Wso#&?<}3roIvWnyLHxhMh)E^ta&0k z-4_WL=|K^2weBUBU9pzO@STm{#5NT%~9} z%L!UKj|`BT__wwCn<;QUX z+KFyAYtYv1GengYKN9@1vb}C~P3SQ$1)dV(-??kCECAJCm#)ok2``W2aBLd@I8AJD zP92@8OXT-_UiVZx_Gp^At>Z~>EMVN>KVBD(d>b1)1XKx{DAlQyNj zo)JR(j7C*2FP^KW%+Ict&YLO!jb~LBaFTwS?fJ1v0TTZ1kq|6(1la;h!MQz$PV7Ck zj#-ADn=PuDhbW&U#JsaiqdvV5nJ&cD;{K;ksS@X$;Ru&mwUbxgYu(6?e?yLGnqo95 zvdYKlc1s_$+oJQYN##F0?Gdt-CZw*~UO%bj^RXvJ#zy+y+wZdxxo2T74I6*@qCG#J zQumV@DN{ibr$2;Ca+s1D;NYTJ``Y~Q+SJ$0*5&NMJ3TKK9Orj|!$f~C zI8H4jr%`7m;0$2|dG~3kbwj|?t9l#%$DwShBGM>BmuCWHk%99Z=+iMzy#*smn%+>Y zIJ2a`XjJ9ukSL0lBh&$mYfEA@cXlWghx)d}ML9bbGFCOIfr5^ICZ|T-SB+0(Rgiar ze1){vkNL}P6$;(oszXoTY<}a|f=${89RFL@-ILh$+_-kL99EjJ#Bz{<+ZH%NB3}?*!DtH+bTppVV5TWk41^9?cv0kYEp2R8y7Kr>(+XS<=t( zXb@x0R55J6uR}ydJu;RW=BM3~8`Yv3PJv}lg=oNHF%g#1b;38)9aciSE45EA z|A@009UGH8dLZlV3Ta^?vd4veL+r1)@nkm4s${+P#d{$e zW^bJ(e)<5#O{=XU_`(Pmd&=C_@7n}hAl?|`bC>cSMqI*H7}M@?3%h@BPwVUb6aKX@ z>heVvRv5^0Bnp#^t9vwaO-kQ+R)}7VzCF`*@e8Wb$pGJqE z;^6w(D*jpJS87e#I~Z?!^lBq`If0y;JfJZ>C#CN1c7g2o#LkFQrO=TxmhP3|Nlh4v zc4nrl%MqN1)W?1YA_rSSXx+xMV-Ni;xDE>n<~vXD11S@Vpl?VzCo_0{ry~vHaxZZ? zo$>8-HN|n?``)5k=!5Ip zNb8ilEZ`hdgEn!-N7qNmJKdH%v8_%m`Bpelo6uy-vn`R89WOQC-klcss=^B6qa(95 zylQa68Tv2c?R6cNS;Ll86LG!mm(KuqbwUo;l7e4+J(xN2#p)2{l z;I{gR`~_o8)X~)6PcovkXl(3#b|sr2e}8M_M`P=okLXS#uN@Qf9%<_k4F*e2Z5L?< z7T~mh`zxaW7e~t7X_Z$m(hWQxMJ6!DBFA3F;Np_`_!#n9z4tw7O1h^sxy>r$bnoQN zjR8R=bze-ULW9lf07{S~7DFg*NzchrSCUd6v$#!tx3{CX;?0etmpp8k+zVXmglp9w zuM%fAQqO6BaON%YI)9m8rIWvBiteePB^VNnNEXR?SP99@lf`jipXZvc78Sl}gJ4JV zSGF5Vpg@bd$Y zcl&PJgMYa5{I8t4w|f`ULnn>^JFTpdSoy^X)7|IGfg^imr$5-y#crwFhOyrWa`ML7 zzoG&YfR>!zB6;LRu`5#-4VI#cCr6^r^IGBev-8`0Q3++c#NZp;rQ@Vn$M_BeAb7dl zi_DHKVCLfWYV;!7i>B;OGic2_A{tKh>$g^9%En7SNlBG;bXz0{9_4Mz)jM<2uDZYE zBNWg62sG=FZS1$GqQJTPJ53bllgSIF$FI6cvx{-OuxPrH!e;|wj1VKL)BsQ=*%|Fj zP!l?cKG_b#{4DE3YQhLP$maCOTsH;)(8B|J#zK+2XBcL~!nPn^@ReAZwfS%j#on9m zfausye!_)yj&OFH_y%rh_2vS0Ri`#-z=XvrE8k>lvIx7xhPcka{l-B{Bs!oOIYg)kb{DWv0d0v%TD$4jlc@V zD_Lfec8z)mR|IecoGLG?7}K+NudYFX$x|FyR(7m*w}P9^K+bG-JPU*4e?Z%S06kIV zV%K*f#IBqMXCPj XWE5AHfp?(@3-}ltT3kTsU%T^P?7O{~ literal 0 HcmV?d00001 diff --git a/tutorials/llm/llama/domain-adaptive-pretraining/code/tokenization_helper.py b/tutorials/llm/llama/domain-adaptive-pretraining/code/tokenization_helper.py new file mode 100755 index 000000000000..6409f7bc041d --- /dev/null +++ b/tutorials/llm/llama/domain-adaptive-pretraining/code/tokenization_helper.py @@ -0,0 +1,399 @@ +import glob +import io +import json +import os +import random +import re +import sys +from collections import Counter + +import jsonlines +import numpy as np +import sentencepiece as spm +import sentencepiece.sentencepiece_model_pb2 as model +import torch +from datasets import Dataset, IterableDataset, load_dataset +from tokenizers import ( + SentencePieceBPETokenizer, + Tokenizer, + decoders, + models, + normalizers, + pre_tokenizers, + processors, + trainers, +) +from transformers import PreTrainedTokenizerFast + +def check_parent_directory_exists(directory_path): + parent_directory = os.path.dirname(directory_path) + if not os.path.exists(parent_directory): + raise FileNotFoundError(f"Parent directory '{parent_directory}' does not exist. Please create it.") + else: + print(f"Parent directory '{parent_directory}' exists.") + + +def flatten(l): + return [item for sublist in l for item in sublist] + + +def get_token_cnt(data_root, tokenizer, batchsize, keys): + """ + Function to get number of tokens generated from a given dataset + + Args: + data_root (str): Path to folder containing data files in jsonl format. + tokenizer (AutoTokenizer): Tokenizer to create tokens from data + batchsize (int): batch size used for the text_iterator that generates of batches of text. + keys (list): Keys/metadata to extract from jsonl files + + Returns: + A new tokenizer of the same type as the original one, trained on data_root + """ + readers = [] + for f in glob.glob(data_root + "**/*.jsonl", recursive=True): + f = open(f, mode="r") + readers.append(jsonlines.Reader(f)) + + def gen(): + data = [] + cnt = 0 + for reader in readers: + for obj in reader: + for key in keys: + data.append(obj[key]) + cnt += 1 + if cnt >= batchsize: + yield data + cnt = 0 + data = [] + if len(data) > 0: + yield data + + ds = IterableDataset.from_generator(gen) + total_cnt = 0 + for d in ds: + ids = tokenizer(d).input_ids # tokenizer.encode(d) + total_cnt += sum([len(i) for i in ids]) + print("total token cnt", total_cnt) + + +def train_tokenizer(data_root, batchsize, vocab_size, tokenizer, keys): + """ + Train tokenizer from scratch and evaluate number of tokens both before and after + Args: + data_root (str): Path to folder containing data files in jsonl format. + batchsize (int): batch size used for the text_iterator that generates of batches of text. + vocab_size (int): The target size of the vocabulary you want for your tokenizer. + tokenizer (AutoTokenizer): Tokenizer to create tokens from data + keys (list): Keys/metadata to extract from jsonl files + + Returns: + A new tokenizer of the same type as the original one, trained on data_root + + """ + print("Before Training: ") + get_token_cnt(data_root, tokenizer, batchsize, keys) + + def gen(): + data = [] + cnt = 0 + for f in glob.glob(data_root + "*.jsonl", recursive=True): + f = open(f, mode="r") + reader = jsonlines.Reader(f) + for obj in reader: + for key in keys: + data.append(obj[key]) + cnt += 1 + if cnt >= batchsize: + yield data + cnt = 0 + data = [] + f.close() + if len(data) > 0: + yield data + + ds = IterableDataset.from_generator(gen) + tokenizer = tokenizer.train_new_from_iterator(ds, vocab_size) + print("After Training: ") + get_token_cnt(data_root, tokenizer, batchsize, keys) + return tokenizer + + +def extend_tokenizer_llama( + data_root, + original_tokenizer_path, + domain_tok_vocab_path, + new_vocab_path, + new_model_path, + old_ebd_path, + new_ebd_path, + split=1, +): + """ + Expand the general-purpose llama tokenizer with the newly identified tokens to get an extended Tokenizer + Args: + data_root (str): Path to general/domain specific data to identify tokens and extend tokenizer + original_tokenizer_path (str): Path to original tokenizer (llama 2 tokenizer downlaoded from hf) + domain_tok_vocab_path (str): Path to domain specific vocab file (created from training a tokenizer from scratch) + new_vocab_path (str): Path to new vocabulary file + new_model_path (str): Path to new/extended tokenizer + old_ebd_path (str): Path to original llama2 embedding weights downlaoded from hf + new_ebd_path (str): Path to new embedding weights (modified due to tokenizer changes) + split (int): Number of splits used for original model weights (model parallelism) + + Returns: + Extended/new model files created and saved in the paths specified below + + """ + keys = ["text"] + occur_limit = 3 + + token_pattern = '[a-zA-Z]' # or [a-zA-Z0-9] + + # Read data from data path and store + readers = [] + for f in glob.glob(data_root + "**/*.jsonl", recursive=True): + f = open(f, mode="r") + readers.append(jsonlines.Reader(f)) + data = [] + for reader in readers: + for obj in reader: + for key in keys: + if key in obj: + data.append(" " + obj[key]) + + # Read domain specific voacb file and analyze added tokens + f = open(domain_tok_vocab_path) + vocab = json.load(f) + print("Domain vocab size:", len(vocab)) + + tokens = [] + drop_tokens = [] + print("token pattern: ", token_pattern) + for v in vocab: + if re.search(token_pattern, v): + tokens.append(v.replace("Ġ", "▁")) + else: + drop_tokens.append(v) + print("Num of added tokens and dropped tokens", len(tokens), len(drop_tokens)) + + m = model.ModelProto() + m.ParseFromString(open(original_tokenizer_path, 'rb').read()) + print(f'Original model pieces: {len(m.pieces)}') + print(m.trainer_spec) + ori_vol = [] + for piece in m.pieces: + ori_vol.append(piece.piece) + print("original vocab size: ", len(ori_vol)) + ori_vol = set(ori_vol) + data = set(data) + + new_tokens = [] + for token in tokens: + if token not in ori_vol: + token1 = token.replace("▁", " ") + occur_cnt = 0 + flag = True + for s in data: + if token1 in s: + occur_cnt += 1 + if occur_cnt > occur_limit: + flag = False + break + if flag: + new_tokens.append(token) + print("new token cnt: ", len(new_tokens)) + + normal_cnt = len(new_tokens) + dummy_cnt = (len(new_tokens) // 1024 + 1) * 1024 - len(new_tokens) + add_cnt = normal_cnt + dummy_cnt + print("add token cnt: ", add_cnt) + print("add normal token cnt: ", normal_cnt) + print("add dummy token cnt: ", dummy_cnt) + assert dummy_cnt >= 3, "should be at least 3 extra tokens for finetuning" + + dummy_tokens = [] + for i in range(dummy_cnt): + dummy_tokens.append(f"") + + record = [] + N = len(m.pieces) + for i, sym in enumerate(new_tokens): + new_sym = m.SentencePiece() + new_sym.piece = sym + new_sym.score = 0.0 # default score for USER_DEFINED + new_sym.type = 4 # type value for USER_DEFINED + m.pieces.insert(N + i, new_sym) # position after default control symbols ("", "", "") + record.append([sym, N + i]) + + N = len(m.pieces) + for i, sym in enumerate(dummy_tokens): + new_sym = m.SentencePiece() + new_sym.piece = sym + new_sym.score = 0.0 # default score for USER_DEFINED + new_sym.type = 4 # type value for USER_DEFINED + m.pieces.insert(N + i, new_sym) # position after default control symbols ("", "", "") + record.append([sym, N + i]) + + print(f'New model pieces: {len(m.pieces)}') + print(m.trainer_spec) + + check_parent_directory_exists(new_vocab_path) + with open(new_vocab_path, "w", encoding="utf8") as fp: + json.dump(record, fp) + + check_parent_directory_exists(new_model_path) + with open(new_model_path, 'wb') as f: + f.write(m.SerializeToString()) + + if split > 1: + old_ebd_paths = [] + for f in glob.glob(old_ebd_path + "/*.pt"): + old_ebd_paths.append(f) + + def myFunc(s): + return int(s.split("embedding_")[-1].split(".")[0]) ### embedding_0.pt + + old_ebd_paths.sort(key=myFunc) + word_embeddings = [] + output_layers = [] + for f in old_ebd_paths: + temp = torch.load(f) + word_embeddings.append(temp['word_embeddings']) + output_layers.append(temp['output_layer']) + word_embedding = torch.cat(word_embeddings, dim=1) + output_layer = torch.cat(output_layers, dim=0) + print("word_embedding shape: ", word_embedding.shape) + print("output_layer shape: ", output_layer.shape) + + _, N = word_embedding.shape + add_weight = torch.zeros(add_cnt, N) + word_embedding = torch.cat((word_embedding, add_weight), 0) + else: + old_ebd = torch.load(old_ebd_path) + _, N = old_ebd['word_embeddings'].shape + add_weight = torch.zeros(add_cnt, N) + old_ebd['word_embeddings'] = torch.cat((old_ebd['word_embeddings'], add_weight), 0) + + if split > 1: + _, M = output_layer.shape + add_out = torch.zeros(add_cnt, M) + output_layer = torch.cat((output_layer, add_out), 0) + else: + _, M = old_ebd['output_layer'].shape + add_out = torch.zeros(add_cnt, M) + old_ebd['output_layer'] = torch.cat((old_ebd['output_layer'], add_out), 0) + + sp = spm.SentencePieceProcessor() + sp.load(original_tokenizer_path) + + for r in record: + token = r[0] + idx = r[1] + ids = sp.encode_as_ids(token) + if split > 1: + word_embedding[idx] = torch.mean(word_embedding[ids], dim=0) + output_layer[idx] = torch.mean(output_layer[ids], dim=0) + else: + old_ebd['word_embeddings'][idx] = torch.mean(old_ebd['word_embeddings'][ids], dim=0) + old_ebd['output_layer'][idx] = torch.mean(old_ebd['output_layer'][ids], dim=0) + + if split > 1: + vocab_size, dimension = word_embedding.shape + split_dimension = dimension // (split) + split_vocab_size = vocab_size // split + prefix = new_ebd_path + "/embedding_" + for i in range(split): + start = i * split_dimension + end = (i + 1) * split_dimension + st = i * split_vocab_size + ed = (i + 1) * split_vocab_size + save_name = prefix + f"{i}" + ".pt" + temp = {} + temp['word_embeddings'] = word_embedding[:, start:end] # split word_embedding + temp['output_layer'] = output_layer[st:ed, :] # split output_layer + check_parent_directory_exists(save_name) + torch.save(temp, save_name) + else: + torch.save(old_ebd, new_ebd_path + str(len(m.pieces)) + ".pt") + + print("Completed saving new embeddings") + + +def analyze_token_usage(data_root, tokenizer_path, batchsize, keys, save_path): + """ + Function to analyze domain tokens using frequency analysis + Args: + data_root (str): Path to general/domain specific data to identify tokens + tokenizer_path (str): Path to original tokenizer (llama 2 tokenizer downlaoded from hf) + batchsize (int): batch size used for the text_iterator that generates of batches of text. + keys (list): Keys/metadata to extract from jsonl files + save_path (str): path to save token usage frequency analysis results + + Returns: + None, saves frequency analysis results to the provided path + + """ + extra_id = 32000 + sp = spm.SentencePieceProcessor() + sp.load(tokenizer_path) + + vocab_size = sp.get_piece_size() + print("vocab_size: ", vocab_size) + results = {} + + for name in glob.glob(data_root + "**/*.jsonl", recursive=True): + readers = [] + f = open(name, mode="r") + readers.append(jsonlines.Reader(f)) + + def gen(): + data = [] + cnt = 0 + for reader in readers: + for obj in reader: + for key in keys: + data.append(obj[key]) + cnt += 1 + if cnt >= batchsize: + yield data + cnt = 0 + data = [] + if len(data) > 0: + yield data + + ds = IterableDataset.from_generator(gen) + cnt_np = np.zeros(vocab_size) + for d in ds: + ids = sp.encode(d) + ids = flatten(ids) + counts = Counter(ids) + for key in counts: + cnt_np[key] += counts[key] + ori_cnt = cnt_np[0:extra_id].sum() + new_cnt = cnt_np[extra_id:].sum() + total_cnt = ori_cnt + new_cnt + print("ori cnt and new cnt: ", ori_cnt, new_cnt) + indices = np.flip(cnt_np.ravel().argsort()[-vocab_size:]) + flag = indices >= extra_id + cnts = cnt_np[indices] + old_freq = [] + new_freq = [] + for i in range(len(indices)): + if cnts[i] < 1: + break + id = indices[i] + if flag[i]: + new_freq.append([int(id), str(sp.id_to_piece(int(id))), int(flag[i]), int(cnts[i])]) + else: + old_freq.append([int(id), str(sp.id_to_piece(int(id))), int(flag[i]), int(cnts[i])]) + results[name] = {} + results[name]["ori_cnt"] = [int(ori_cnt), float(ori_cnt / total_cnt)] + results[name]["new_cnt"] = [int(new_cnt), float(new_cnt / total_cnt)] + results[name]["old_freq"] = old_freq + results[name]["new_freq"] = new_freq + f.close() + + with open(save_path, "w") as outfile: + json.dump(results, outfile) diff --git a/tutorials/llm/llama/domain-adaptive-pretraining/code/util.py b/tutorials/llm/llama/domain-adaptive-pretraining/code/util.py new file mode 100755 index 000000000000..50aae500bdd0 --- /dev/null +++ b/tutorials/llm/llama/domain-adaptive-pretraining/code/util.py @@ -0,0 +1,65 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import torch + + +def check_directory_exists(directory): + if os.path.isdir(directory): + print(f"Directory '{directory}' exists") + else: + raise FileNotFoundError(f"The directory '{directory}' does not exist. Please create it.") + + +def load_weights(load_path, save_path): + """ + This function loads llama2 weights (hugging face) and converts it to a Dict format suitable for NeMo + + Args: + load_path (str): Path to llama2 weights downlaoded from hugging face + save_path (str): Path to save modified dictionary containing the weights. + + Returns: + None + + """ + model_type = "llama2" + for i in range(8): + state_dict = torch.load(f"{load_path}/consolidated.0{i}.pth") + batch_dict = {} + if model_type == "llama2": + batch_dict['word_embeddings'] = state_dict['tok_embeddings.weight'] + batch_dict['output_layer'] = state_dict['output.weight'] + else: + batch_dict['word_embeddings'] = state_dict['model']['embedding.word_embeddings.weight'] # embedding layer + batch_dict['output_layer'] = state_dict['model']['output_layer.weight'] # output layer + torch.save(batch_dict, f'{save_path}/embedding_{i}.pt') + + +def merge_embed(old_embd_path, new_embd_path, save_path): + "Function to merge embeddings and convert back to hugging face format" + model_type = "llama2" + for i in range(8): + state_dict = torch.load(f"{old_embd_path}/consolidated.0{i}.pth") + batch_dict = torch.load(f'{new_embd_path}/embedding_{i}.pt') + if model_type == "llama2": + state_dict['output.weight'] = batch_dict['output_layer'] + state_dict['tok_embeddings.weight'] = batch_dict['word_embeddings'] + else: + state_dict['tok_embeddings.weight'] = batch_dict['word_embeddings'] + state_dict['output.weight'] = batch_dict['output_layer'] + check_directory_exists(save_path) + torch.save(state_dict, f"{save_path}/consolidated.0{i}.pth") \ No newline at end of file From e55d26a5d47723c28e4c0be43dad916f295c92d8 Mon Sep 17 00:00:00 2001 From: Janaki Date: Thu, 6 Feb 2025 09:12:57 -0800 Subject: [PATCH 2/9] DAPT with NeMo 2.0 --- tutorials/llm/{llama-3 => llama}/README.rst | 0 .../llm/{llama-3 => llama}/biomedical-qa/README.rst | 0 .../biomedical-qa/img/e2e-lora-train-and-deploy.png | Bin .../biomedical-qa/llama3-lora-deploy-nim.ipynb | 0 .../biomedical-qa/llama3-lora-nemofw.ipynb | 0 .../{llama-3 => llama}/nemo2-sft-peft/README.rst | 0 .../nemo2-sft-peft/nemo2-peft.ipynb | 0 .../nemo2-sft-peft/nemo2-sft.ipynb | 0 .../pruning-distillation/01_data_preparation.ipynb | 0 .../02_teacher_finetuning.ipynb | 0 .../pruning-distillation/03_a_depth_pruning.ipynb | 0 .../pruning-distillation/03_b_width_pruning.ipynb | 0 .../04_a_distilling_depth_pruned_student.ipynb | 0 .../04_b_distilling_width_pruned_student.ipynb | 0 .../pruning-distillation/05_display_results.ipynb | 0 .../pruning-distillation/README.rst | 0 .../pruning-distillation/introduction.ipynb | 0 .../sdg-law-title-generation/README.rst | 0 .../img/e2e-lora-train-and-deploy.png | Bin .../llama3-sdg-lora-deploy-nim.ipynb | 0 .../llama3-sdg-lora-nemofw.ipynb | 0 .../llm/{llama-3 => llama}/slimpajama/README.md | 0 .../{llama-3 => llama}/slimpajama/data/concat.sh | 0 .../{llama-3 => llama}/slimpajama/data/download.py | 0 .../{llama-3 => llama}/slimpajama/data/extract.py | 0 .../slimpajama/data/preprocess.py | 0 .../slimpajama/data_pipeline.ipynb | 0 .../{llama-3 => llama}/slimpajama/data_pipeline.py | 0 .../{llama-3 => llama}/slimpajama/pretraining.ipynb | 0 29 files changed, 0 insertions(+), 0 deletions(-) rename tutorials/llm/{llama-3 => llama}/README.rst (100%) rename tutorials/llm/{llama-3 => llama}/biomedical-qa/README.rst (100%) rename tutorials/llm/{llama-3 => llama}/biomedical-qa/img/e2e-lora-train-and-deploy.png (100%) rename tutorials/llm/{llama-3 => llama}/biomedical-qa/llama3-lora-deploy-nim.ipynb (100%) rename tutorials/llm/{llama-3 => llama}/biomedical-qa/llama3-lora-nemofw.ipynb (100%) rename tutorials/llm/{llama-3 => llama}/nemo2-sft-peft/README.rst (100%) rename tutorials/llm/{llama-3 => llama}/nemo2-sft-peft/nemo2-peft.ipynb (100%) rename tutorials/llm/{llama-3 => llama}/nemo2-sft-peft/nemo2-sft.ipynb (100%) rename tutorials/llm/{llama-3 => llama}/pruning-distillation/01_data_preparation.ipynb (100%) rename tutorials/llm/{llama-3 => llama}/pruning-distillation/02_teacher_finetuning.ipynb (100%) rename tutorials/llm/{llama-3 => llama}/pruning-distillation/03_a_depth_pruning.ipynb (100%) rename tutorials/llm/{llama-3 => llama}/pruning-distillation/03_b_width_pruning.ipynb (100%) rename tutorials/llm/{llama-3 => llama}/pruning-distillation/04_a_distilling_depth_pruned_student.ipynb (100%) rename tutorials/llm/{llama-3 => llama}/pruning-distillation/04_b_distilling_width_pruned_student.ipynb (100%) rename tutorials/llm/{llama-3 => llama}/pruning-distillation/05_display_results.ipynb (100%) rename tutorials/llm/{llama-3 => llama}/pruning-distillation/README.rst (100%) rename tutorials/llm/{llama-3 => llama}/pruning-distillation/introduction.ipynb (100%) rename tutorials/llm/{llama-3 => llama}/sdg-law-title-generation/README.rst (100%) rename tutorials/llm/{llama-3 => llama}/sdg-law-title-generation/img/e2e-lora-train-and-deploy.png (100%) rename tutorials/llm/{llama-3 => llama}/sdg-law-title-generation/llama3-sdg-lora-deploy-nim.ipynb (100%) rename tutorials/llm/{llama-3 => llama}/sdg-law-title-generation/llama3-sdg-lora-nemofw.ipynb (100%) rename tutorials/llm/{llama-3 => llama}/slimpajama/README.md (100%) rename tutorials/llm/{llama-3 => llama}/slimpajama/data/concat.sh (100%) rename tutorials/llm/{llama-3 => llama}/slimpajama/data/download.py (100%) rename tutorials/llm/{llama-3 => llama}/slimpajama/data/extract.py (100%) rename tutorials/llm/{llama-3 => llama}/slimpajama/data/preprocess.py (100%) rename tutorials/llm/{llama-3 => llama}/slimpajama/data_pipeline.ipynb (100%) rename tutorials/llm/{llama-3 => llama}/slimpajama/data_pipeline.py (100%) rename tutorials/llm/{llama-3 => llama}/slimpajama/pretraining.ipynb (100%) diff --git a/tutorials/llm/llama-3/README.rst b/tutorials/llm/llama/README.rst similarity index 100% rename from tutorials/llm/llama-3/README.rst rename to tutorials/llm/llama/README.rst diff --git a/tutorials/llm/llama-3/biomedical-qa/README.rst b/tutorials/llm/llama/biomedical-qa/README.rst similarity index 100% rename from tutorials/llm/llama-3/biomedical-qa/README.rst rename to tutorials/llm/llama/biomedical-qa/README.rst diff --git a/tutorials/llm/llama-3/biomedical-qa/img/e2e-lora-train-and-deploy.png b/tutorials/llm/llama/biomedical-qa/img/e2e-lora-train-and-deploy.png similarity index 100% rename from tutorials/llm/llama-3/biomedical-qa/img/e2e-lora-train-and-deploy.png rename to tutorials/llm/llama/biomedical-qa/img/e2e-lora-train-and-deploy.png diff --git a/tutorials/llm/llama-3/biomedical-qa/llama3-lora-deploy-nim.ipynb b/tutorials/llm/llama/biomedical-qa/llama3-lora-deploy-nim.ipynb similarity index 100% rename from tutorials/llm/llama-3/biomedical-qa/llama3-lora-deploy-nim.ipynb rename to tutorials/llm/llama/biomedical-qa/llama3-lora-deploy-nim.ipynb diff --git a/tutorials/llm/llama-3/biomedical-qa/llama3-lora-nemofw.ipynb b/tutorials/llm/llama/biomedical-qa/llama3-lora-nemofw.ipynb similarity index 100% rename from tutorials/llm/llama-3/biomedical-qa/llama3-lora-nemofw.ipynb rename to tutorials/llm/llama/biomedical-qa/llama3-lora-nemofw.ipynb diff --git a/tutorials/llm/llama-3/nemo2-sft-peft/README.rst b/tutorials/llm/llama/nemo2-sft-peft/README.rst similarity index 100% rename from tutorials/llm/llama-3/nemo2-sft-peft/README.rst rename to tutorials/llm/llama/nemo2-sft-peft/README.rst diff --git a/tutorials/llm/llama-3/nemo2-sft-peft/nemo2-peft.ipynb b/tutorials/llm/llama/nemo2-sft-peft/nemo2-peft.ipynb similarity index 100% rename from tutorials/llm/llama-3/nemo2-sft-peft/nemo2-peft.ipynb rename to tutorials/llm/llama/nemo2-sft-peft/nemo2-peft.ipynb diff --git a/tutorials/llm/llama-3/nemo2-sft-peft/nemo2-sft.ipynb b/tutorials/llm/llama/nemo2-sft-peft/nemo2-sft.ipynb similarity index 100% rename from tutorials/llm/llama-3/nemo2-sft-peft/nemo2-sft.ipynb rename to tutorials/llm/llama/nemo2-sft-peft/nemo2-sft.ipynb diff --git a/tutorials/llm/llama-3/pruning-distillation/01_data_preparation.ipynb b/tutorials/llm/llama/pruning-distillation/01_data_preparation.ipynb similarity index 100% rename from tutorials/llm/llama-3/pruning-distillation/01_data_preparation.ipynb rename to tutorials/llm/llama/pruning-distillation/01_data_preparation.ipynb diff --git a/tutorials/llm/llama-3/pruning-distillation/02_teacher_finetuning.ipynb b/tutorials/llm/llama/pruning-distillation/02_teacher_finetuning.ipynb similarity index 100% rename from tutorials/llm/llama-3/pruning-distillation/02_teacher_finetuning.ipynb rename to tutorials/llm/llama/pruning-distillation/02_teacher_finetuning.ipynb diff --git a/tutorials/llm/llama-3/pruning-distillation/03_a_depth_pruning.ipynb b/tutorials/llm/llama/pruning-distillation/03_a_depth_pruning.ipynb similarity index 100% rename from tutorials/llm/llama-3/pruning-distillation/03_a_depth_pruning.ipynb rename to tutorials/llm/llama/pruning-distillation/03_a_depth_pruning.ipynb diff --git a/tutorials/llm/llama-3/pruning-distillation/03_b_width_pruning.ipynb b/tutorials/llm/llama/pruning-distillation/03_b_width_pruning.ipynb similarity index 100% rename from tutorials/llm/llama-3/pruning-distillation/03_b_width_pruning.ipynb rename to tutorials/llm/llama/pruning-distillation/03_b_width_pruning.ipynb diff --git a/tutorials/llm/llama-3/pruning-distillation/04_a_distilling_depth_pruned_student.ipynb b/tutorials/llm/llama/pruning-distillation/04_a_distilling_depth_pruned_student.ipynb similarity index 100% rename from tutorials/llm/llama-3/pruning-distillation/04_a_distilling_depth_pruned_student.ipynb rename to tutorials/llm/llama/pruning-distillation/04_a_distilling_depth_pruned_student.ipynb diff --git a/tutorials/llm/llama-3/pruning-distillation/04_b_distilling_width_pruned_student.ipynb b/tutorials/llm/llama/pruning-distillation/04_b_distilling_width_pruned_student.ipynb similarity index 100% rename from tutorials/llm/llama-3/pruning-distillation/04_b_distilling_width_pruned_student.ipynb rename to tutorials/llm/llama/pruning-distillation/04_b_distilling_width_pruned_student.ipynb diff --git a/tutorials/llm/llama-3/pruning-distillation/05_display_results.ipynb b/tutorials/llm/llama/pruning-distillation/05_display_results.ipynb similarity index 100% rename from tutorials/llm/llama-3/pruning-distillation/05_display_results.ipynb rename to tutorials/llm/llama/pruning-distillation/05_display_results.ipynb diff --git a/tutorials/llm/llama-3/pruning-distillation/README.rst b/tutorials/llm/llama/pruning-distillation/README.rst similarity index 100% rename from tutorials/llm/llama-3/pruning-distillation/README.rst rename to tutorials/llm/llama/pruning-distillation/README.rst diff --git a/tutorials/llm/llama-3/pruning-distillation/introduction.ipynb b/tutorials/llm/llama/pruning-distillation/introduction.ipynb similarity index 100% rename from tutorials/llm/llama-3/pruning-distillation/introduction.ipynb rename to tutorials/llm/llama/pruning-distillation/introduction.ipynb diff --git a/tutorials/llm/llama-3/sdg-law-title-generation/README.rst b/tutorials/llm/llama/sdg-law-title-generation/README.rst similarity index 100% rename from tutorials/llm/llama-3/sdg-law-title-generation/README.rst rename to tutorials/llm/llama/sdg-law-title-generation/README.rst diff --git a/tutorials/llm/llama-3/sdg-law-title-generation/img/e2e-lora-train-and-deploy.png b/tutorials/llm/llama/sdg-law-title-generation/img/e2e-lora-train-and-deploy.png similarity index 100% rename from tutorials/llm/llama-3/sdg-law-title-generation/img/e2e-lora-train-and-deploy.png rename to tutorials/llm/llama/sdg-law-title-generation/img/e2e-lora-train-and-deploy.png diff --git a/tutorials/llm/llama-3/sdg-law-title-generation/llama3-sdg-lora-deploy-nim.ipynb b/tutorials/llm/llama/sdg-law-title-generation/llama3-sdg-lora-deploy-nim.ipynb similarity index 100% rename from tutorials/llm/llama-3/sdg-law-title-generation/llama3-sdg-lora-deploy-nim.ipynb rename to tutorials/llm/llama/sdg-law-title-generation/llama3-sdg-lora-deploy-nim.ipynb diff --git a/tutorials/llm/llama-3/sdg-law-title-generation/llama3-sdg-lora-nemofw.ipynb b/tutorials/llm/llama/sdg-law-title-generation/llama3-sdg-lora-nemofw.ipynb similarity index 100% rename from tutorials/llm/llama-3/sdg-law-title-generation/llama3-sdg-lora-nemofw.ipynb rename to tutorials/llm/llama/sdg-law-title-generation/llama3-sdg-lora-nemofw.ipynb diff --git a/tutorials/llm/llama-3/slimpajama/README.md b/tutorials/llm/llama/slimpajama/README.md similarity index 100% rename from tutorials/llm/llama-3/slimpajama/README.md rename to tutorials/llm/llama/slimpajama/README.md diff --git a/tutorials/llm/llama-3/slimpajama/data/concat.sh b/tutorials/llm/llama/slimpajama/data/concat.sh similarity index 100% rename from tutorials/llm/llama-3/slimpajama/data/concat.sh rename to tutorials/llm/llama/slimpajama/data/concat.sh diff --git a/tutorials/llm/llama-3/slimpajama/data/download.py b/tutorials/llm/llama/slimpajama/data/download.py similarity index 100% rename from tutorials/llm/llama-3/slimpajama/data/download.py rename to tutorials/llm/llama/slimpajama/data/download.py diff --git a/tutorials/llm/llama-3/slimpajama/data/extract.py b/tutorials/llm/llama/slimpajama/data/extract.py similarity index 100% rename from tutorials/llm/llama-3/slimpajama/data/extract.py rename to tutorials/llm/llama/slimpajama/data/extract.py diff --git a/tutorials/llm/llama-3/slimpajama/data/preprocess.py b/tutorials/llm/llama/slimpajama/data/preprocess.py similarity index 100% rename from tutorials/llm/llama-3/slimpajama/data/preprocess.py rename to tutorials/llm/llama/slimpajama/data/preprocess.py diff --git a/tutorials/llm/llama-3/slimpajama/data_pipeline.ipynb b/tutorials/llm/llama/slimpajama/data_pipeline.ipynb similarity index 100% rename from tutorials/llm/llama-3/slimpajama/data_pipeline.ipynb rename to tutorials/llm/llama/slimpajama/data_pipeline.ipynb diff --git a/tutorials/llm/llama-3/slimpajama/data_pipeline.py b/tutorials/llm/llama/slimpajama/data_pipeline.py similarity index 100% rename from tutorials/llm/llama-3/slimpajama/data_pipeline.py rename to tutorials/llm/llama/slimpajama/data_pipeline.py diff --git a/tutorials/llm/llama-3/slimpajama/pretraining.ipynb b/tutorials/llm/llama/slimpajama/pretraining.ipynb similarity index 100% rename from tutorials/llm/llama-3/slimpajama/pretraining.ipynb rename to tutorials/llm/llama/slimpajama/pretraining.ipynb From 5f31dcf6b1db15efb60605f81c1b44470ba2a9a6 Mon Sep 17 00:00:00 2001 From: jvamaraju Date: Thu, 6 Feb 2025 17:17:44 +0000 Subject: [PATCH 3/9] Apply isort and black reformatting Signed-off-by: jvamaraju --- .../code/extend_tokenizer_utils.py | 2 +- .../code/get_high_freq_tokens.py | 6 ++++-- .../code/tokenization_helper.py | 11 ++++++----- .../llama/domain-adaptive-pretraining/code/util.py | 6 +++--- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/tutorials/llm/llama/domain-adaptive-pretraining/code/extend_tokenizer_utils.py b/tutorials/llm/llama/domain-adaptive-pretraining/code/extend_tokenizer_utils.py index a1ef53920d65..ba87b68bd59c 100755 --- a/tutorials/llm/llama/domain-adaptive-pretraining/code/extend_tokenizer_utils.py +++ b/tutorials/llm/llama/domain-adaptive-pretraining/code/extend_tokenizer_utils.py @@ -247,7 +247,7 @@ def myFunc(s): save_name = prefix + f"{i}" + ".pt" temp = {} temp['word_embeddings'] = word_embedding[:, start:end] # split word_embedding - temp['output_layer'] = output_layer[st:ed, :] # split output_layer + temp['output_layer'] = output_layer[st:ed, :] # split output_layer torch.save(temp, save_name) print("Completed saving new embeddings") diff --git a/tutorials/llm/llama/domain-adaptive-pretraining/code/get_high_freq_tokens.py b/tutorials/llm/llama/domain-adaptive-pretraining/code/get_high_freq_tokens.py index 70fcce20b9f9..0c8401cb69d2 100755 --- a/tutorials/llm/llama/domain-adaptive-pretraining/code/get_high_freq_tokens.py +++ b/tutorials/llm/llama/domain-adaptive-pretraining/code/get_high_freq_tokens.py @@ -1,9 +1,9 @@ import json import sys +from bisect import bisect_left import matplotlib.pyplot as plt import numpy as np -from bisect import bisect_left # + @@ -28,7 +28,8 @@ def binary_search(arr, low, high, bar=0.98): return binary_search(arr, low, mid - 1, bar) else: return low - + + def binary_search2(arr, low, high, bar=0.98): arr_csum = np.cumsum(arr) total = arr.sum() @@ -43,6 +44,7 @@ def binary_search2(arr, low, high, bar=0.98): # - + def get_high_freq_tokens(token_usage_path, high_freq_tokens_path, p_th=0.98): """ Function to identify high frequency tokens from previous frequency analysis based on cutoff threshold. Selects the top-K tokens in a way that their cumulative frequency accounts for approximately 98%. diff --git a/tutorials/llm/llama/domain-adaptive-pretraining/code/tokenization_helper.py b/tutorials/llm/llama/domain-adaptive-pretraining/code/tokenization_helper.py index 6409f7bc041d..e7a0c29846d7 100755 --- a/tutorials/llm/llama/domain-adaptive-pretraining/code/tokenization_helper.py +++ b/tutorials/llm/llama/domain-adaptive-pretraining/code/tokenization_helper.py @@ -23,7 +23,8 @@ processors, trainers, ) -from transformers import PreTrainedTokenizerFast +from transformers import PreTrainedTokenizerFast + def check_parent_directory_exists(directory_path): parent_directory = os.path.dirname(directory_path) @@ -238,7 +239,7 @@ def extend_tokenizer_llama( print(f'New model pieces: {len(m.pieces)}') print(m.trainer_spec) - + check_parent_directory_exists(new_vocab_path) with open(new_vocab_path, "w", encoding="utf8") as fp: json.dump(record, fp) @@ -311,8 +312,8 @@ def myFunc(s): ed = (i + 1) * split_vocab_size save_name = prefix + f"{i}" + ".pt" temp = {} - temp['word_embeddings'] = word_embedding[:, start:end] # split word_embedding - temp['output_layer'] = output_layer[st:ed, :] # split output_layer + temp['word_embeddings'] = word_embedding[:, start:end] # split word_embedding + temp['output_layer'] = output_layer[st:ed, :] # split output_layer check_parent_directory_exists(save_name) torch.save(temp, save_name) else: @@ -335,7 +336,7 @@ def analyze_token_usage(data_root, tokenizer_path, batchsize, keys, save_path): None, saves frequency analysis results to the provided path """ - extra_id = 32000 + extra_id = 32000 sp = spm.SentencePieceProcessor() sp.load(tokenizer_path) diff --git a/tutorials/llm/llama/domain-adaptive-pretraining/code/util.py b/tutorials/llm/llama/domain-adaptive-pretraining/code/util.py index 50aae500bdd0..a0fece98cfce 100755 --- a/tutorials/llm/llama/domain-adaptive-pretraining/code/util.py +++ b/tutorials/llm/llama/domain-adaptive-pretraining/code/util.py @@ -22,8 +22,8 @@ def check_directory_exists(directory): print(f"Directory '{directory}' exists") else: raise FileNotFoundError(f"The directory '{directory}' does not exist. Please create it.") - - + + def load_weights(load_path, save_path): """ This function loads llama2 weights (hugging face) and converts it to a Dict format suitable for NeMo @@ -62,4 +62,4 @@ def merge_embed(old_embd_path, new_embd_path, save_path): state_dict['tok_embeddings.weight'] = batch_dict['word_embeddings'] state_dict['output.weight'] = batch_dict['output_layer'] check_directory_exists(save_path) - torch.save(state_dict, f"{save_path}/consolidated.0{i}.pth") \ No newline at end of file + torch.save(state_dict, f"{save_path}/consolidated.0{i}.pth") From 5fb083eb0af111575362abbcb2c644a9c5566c78 Mon Sep 17 00:00:00 2001 From: Janaki Date: Thu, 6 Feb 2025 09:20:18 -0800 Subject: [PATCH 4/9] Deleting file not needed --- .../code/__init__.py | 71 ------------------- 1 file changed, 71 deletions(-) delete mode 100644 tutorials/llm/llama/domain-adaptive-pretraining/code/__init__.py diff --git a/tutorials/llm/llama/domain-adaptive-pretraining/code/__init__.py b/tutorials/llm/llama/domain-adaptive-pretraining/code/__init__.py deleted file mode 100644 index 2e4094093d80..000000000000 --- a/tutorials/llm/llama/domain-adaptive-pretraining/code/__init__.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from nemo.collections.llm.recipes import ( - llama2_7b, - llama3_8b, - llama3_8b_16k, - llama3_8b_64k, - llama3_70b, - llama3_70b_16k, - llama3_70b_64k, - llama31_405b, - mistral, - mixtral_8x7b, - mixtral_8x7b_16k, - mixtral_8x7b_64k, - mixtral_8x22b, - nemotron, - nemotron3_4b, - nemotron3_8b, - nemotron4_15b, - nemotron4_15b_16k, - nemotron4_15b_64k, - nemotron4_22b, - nemotron4_22b_16k, - nemotron4_22b_64k, - nemotron4_340b, -) -from nemo.collections.llm.recipes.log.default import default_log, default_resume -from nemo.collections.llm.recipes.optim import adam - -__all__ = [ - "llama2_7b", - "llama3_8b", - "llama3_8b_16k", - "llama3_8b_64k", - "llama3_70b", - "llama3_70b_16k", - "llama3_70b_64k", - "llama31_405b", - "mistral", - "mixtral_8x7b", - "mixtral_8x7b_16k", - "mixtral_8x7b_64k", - "mixtral_8x22b", - "nemotron", - "nemotron3_4b", - "nemotron3_8b", - "nemotron4_15b", - "nemotron4_15b_16k", - "nemotron4_15b_64k", - "nemotron4_22b", - "nemotron4_22b_16k", - "nemotron4_22b_64k", - "nemotron4_340b", - "adam", - "default_log", - "default_resume", -] From 4dcfc6c54a0a2996d787e3d284a0d2ca958d35c6 Mon Sep 17 00:00:00 2001 From: jvamaraju Date: Thu, 6 Feb 2025 11:35:49 -0600 Subject: [PATCH 5/9] Update README.md Signed-off-by: jvamaraju --- tutorials/llm/llama/domain-adaptive-pretraining/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutorials/llm/llama/domain-adaptive-pretraining/README.md b/tutorials/llm/llama/domain-adaptive-pretraining/README.md index d2ce82a537d0..2cd24d5ab712 100755 --- a/tutorials/llm/llama/domain-adaptive-pretraining/README.md +++ b/tutorials/llm/llama/domain-adaptive-pretraining/README.md @@ -1,4 +1,4 @@ -# ChipNeMo - Custom tokenization + Domain Adaptive Pre-training on Llama 2 70b with NeMo Framework +# ChipNeMo - Custom tokenization + Domain Adaptive Pre-training on Llama 2 7b with NeMo Framework [ChipNeMo](https://arxiv.org/pdf/2311.00176) is a chip design domain-adapted Large Language Model (LLM). Instead of directly deploying off-the-shelf commercial or open-source LLMs, the paper adopts the following domain adaptation techniques: domain-adaptive tokenization, domain-adaptive continued pre-training, model alignment with domain-specific instructions, and domain-adapted retrieval models. Specifically, Llama 2 foundation models are continually pre-trained with more than 20 billion tokens on domain-specific chip design data, including code and documents. They are then fine-tuned with instruction datasets from design data as well as external sources. Evaluations on the resultant domain-adapted ChipNeMo model demonstrate that domain-adaptive pre-training of language models can lead to superior performance in domain-related downstream tasks compared to their base Llama 2 counterparts, without degradations in generic capabilities. From 5869b7ff5267b03528ee9c71b0d253eb9bab702b Mon Sep 17 00:00:00 2001 From: aastha Date: Thu, 6 Feb 2025 23:47:40 -0800 Subject: [PATCH 6/9] Addressing feedback from PR review for DAPT playbook with nemo 2.0 --- .../domain_adaptive_pretraining_nemo2.0.ipynb | 49 ++++++++++++++----- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/tutorials/llm/llama/domain-adaptive-pretraining/code/domain_adaptive_pretraining_nemo2.0.ipynb b/tutorials/llm/llama/domain-adaptive-pretraining/code/domain_adaptive_pretraining_nemo2.0.ipynb index c6ca48697fa9..9c6a9142035f 100644 --- a/tutorials/llm/llama/domain-adaptive-pretraining/code/domain_adaptive_pretraining_nemo2.0.ipynb +++ b/tutorials/llm/llama/domain-adaptive-pretraining/code/domain_adaptive_pretraining_nemo2.0.ipynb @@ -86,6 +86,28 @@ "* Step 3: Continued pretraining the llama-2-7b model using the prepared data and the custom trained tokenizer (from the previous notebook)." ] }, + { + "cell_type": "markdown", + "id": "115e8b1f", + "metadata": {}, + "source": [ + "# Step 0: Clone the Model Checkpoint\n", + "\n", + "This notebook assumed the model has been cloned from [hugging face](https://huggingface.co/meta-llama/Llama-2-7b-hf) in the mounted directory ```/workspace```" + ] + }, + { + "cell_type": "markdown", + "id": "9bc658bd", + "metadata": {}, + "source": [ + "Clone the model: \n", + "```\n", + "git lfs install\n", + "git clone https://huggingface.co/meta-llama/Llama-2-7b-hf\n", + "```" + ] + }, { "cell_type": "markdown", "id": "ec372453", @@ -358,7 +380,9 @@ "id": "b94e774b", "metadata": {}, "source": [ - "The conversion will generate a ```llama-2-7b.nemo``` which can be used for the continued pretraining using NeMo Toolkit as shown in Step 3 in default ```$NEMO_HOME``` folder." + "The conversion will generate a ```llama-2``` NeMo2 checkpoint directory which can be used for the continued pretraining using NeMo Toolkit as shown in Step 3 in default ```$NEMO_HOME``` folder, unless otherwise specified ```NEMO_HOME``` is set as ```/root/.cache/nemo```\n", + "\n", + "Alternatively, you can directly use ```source=\"meta-llama/Llama2-2-7b-hf\"``` to use the model directly from Hugging Face instead of using the locally downloaded version in ```\\workspace```" ] }, { @@ -409,15 +433,6 @@ "\n", "# Executor for running pretraining \n", "def local_executor_torchrun(nodes: int = 1, devices: int = 1) -> run.LocalExecutor:\n", - " # Env vars for jobs are configured here\n", - " # env_vars = {\n", - " # \"TORCH_NCCL_AVOID_RECORD_STREAMS\": \"1\",\n", - " # \"NCCL_NVLS_ENABLE\": \"0\",\n", - " # \"NVTE_DP_AMAX_REDUCE_INTERVAL\": \"0\",\n", - " # \"NVTE_ASYNC_AMAX_REDUCTION\": \"1\",\n", - " # \"NVTE_FUSED_ATTN\": \"0\",\n", - " # }\n", - " # executor = run.LocalExecutor(ntasks_per_node=devices, launcher=\"torchrun\", env_vars=env_vars)\n", " executor = run.LocalExecutor(ntasks_per_node=devices, launcher=\"torchrun\")\n", " return executor" ] @@ -477,15 +492,25 @@ "recipe.trainer.strategy.tensor_model_parallel_size = 2\n", "recipe.trainer.strategy.pipeline_model_parallel_size = 1\n", "recipe.trainer.strategy.context_parallel_size = 1\n", + "\n", + "# (Optional) Modify the batch size settings\n", "recipe.data.global_batch_size = 8\n", + "recipe.data.micro_batch_size = 1\n", + "\n", + "# (Optional) Modify the checkpoint and log location\n", "recipe.log.log_dir= \"/workspace/logs_01_31\"\n", "\n", + "# (Optional) Modify the learning rate scheudler\n", + "recipe.optim.config.lr = 1e-5\n", + "recipe.optim.lr_scheduler.min_lr = 1e-6\n", + "\n", "# If not configured, the recipe uses mock data for pretraining\n", "recipe.data = data\n", "\n", "# (Optional) Modify the data blends\n", "# recipe.data.paths = [0.2, 'path/to/data1', 0.1, 'path/to/data2']\n", - "# recipe.data.paths = [1, 'preprocessed_data_text_document']" + "# recipe.data.paths = [1, 'preprocessed_data_text_document']\n", + "# (Optional) Modify the TP/PP/CP settings" ] }, { @@ -504,7 +529,7 @@ "outputs": [], "source": [ "# Launch the pretraining job \n", - "executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)\n", + "executor = local_executor_torchrun(devices: int = 1)\n", "run.run(recipe, executor=executor)" ] }, From 10de1e653937c7fd3d027c02801b98f93ac41b9a Mon Sep 17 00:00:00 2001 From: aastha Date: Fri, 7 Feb 2025 09:17:46 -0800 Subject: [PATCH 7/9] Addressing feedback for DAPT with nemo 2.0 --- .../code/domain_adaptive_pretraining_nemo2.0.ipynb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tutorials/llm/llama/domain-adaptive-pretraining/code/domain_adaptive_pretraining_nemo2.0.ipynb b/tutorials/llm/llama/domain-adaptive-pretraining/code/domain_adaptive_pretraining_nemo2.0.ipynb index 9c6a9142035f..490b9c3603c2 100644 --- a/tutorials/llm/llama/domain-adaptive-pretraining/code/domain_adaptive_pretraining_nemo2.0.ipynb +++ b/tutorials/llm/llama/domain-adaptive-pretraining/code/domain_adaptive_pretraining_nemo2.0.ipynb @@ -420,7 +420,7 @@ "# Configure recipe to pre-train based on the default llama-2-7b recipe\n", "def configure_recipe(nodes: int = 1, gpus_per_node: int = 1):\n", " recipe = llm.llama2_7b.pretrain_recipe(\n", - " name=\"llama2_7b_pretraining\",\n", + " name=\"llama2_7b_dapt\",\n", " # Modify based on number of nodes available\n", " num_nodes=nodes,\n", " num_gpus_per_node=gpus_per_node,\n", @@ -432,7 +432,7 @@ " return recipe\n", "\n", "# Executor for running pretraining \n", - "def local_executor_torchrun(nodes: int = 1, devices: int = 1) -> run.LocalExecutor:\n", + "def local_executor_torchrun(devices: int = 1) -> run.LocalExecutor:\n", " executor = run.LocalExecutor(ntasks_per_node=devices, launcher=\"torchrun\")\n", " return executor" ] @@ -509,8 +509,7 @@ "\n", "# (Optional) Modify the data blends\n", "# recipe.data.paths = [0.2, 'path/to/data1', 0.1, 'path/to/data2']\n", - "# recipe.data.paths = [1, 'preprocessed_data_text_document']\n", - "# (Optional) Modify the TP/PP/CP settings" + "# recipe.data.paths = [1, 'preprocessed_data_text_document']" ] }, { From 7e53e27004fd2484cf948ae05de466c5970c6ed5 Mon Sep 17 00:00:00 2001 From: aastha Date: Fri, 7 Feb 2025 10:16:39 -0800 Subject: [PATCH 8/9] Addressing feedback for DAPT with nemo 2.0- local executor --- .../code/domain_adaptive_pretraining_nemo2.0.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutorials/llm/llama/domain-adaptive-pretraining/code/domain_adaptive_pretraining_nemo2.0.ipynb b/tutorials/llm/llama/domain-adaptive-pretraining/code/domain_adaptive_pretraining_nemo2.0.ipynb index 490b9c3603c2..84d3ce6b619d 100644 --- a/tutorials/llm/llama/domain-adaptive-pretraining/code/domain_adaptive_pretraining_nemo2.0.ipynb +++ b/tutorials/llm/llama/domain-adaptive-pretraining/code/domain_adaptive_pretraining_nemo2.0.ipynb @@ -528,7 +528,7 @@ "outputs": [], "source": [ "# Launch the pretraining job \n", - "executor = local_executor_torchrun(devices: int = 1)\n", + "executor = local_executor_torchrun(devices=recipe.trainer.devices)\n", "run.run(recipe, executor=executor)" ] }, From 8c8cf8a97e097cb800c47665a71e27ed6235e7e2 Mon Sep 17 00:00:00 2001 From: Ao Tang Date: Mon, 10 Feb 2025 13:40:01 -0500 Subject: [PATCH 9/9] Add Copyright --- .../code/extend_tokenizer_utils.py | 14 ++++++++++++++ .../code/get_high_freq_tokens.py | 14 ++++++++++++++ .../code/tokenization_helper.py | 14 ++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/tutorials/llm/llama/domain-adaptive-pretraining/code/extend_tokenizer_utils.py b/tutorials/llm/llama/domain-adaptive-pretraining/code/extend_tokenizer_utils.py index ba87b68bd59c..1e6de449e9a4 100755 --- a/tutorials/llm/llama/domain-adaptive-pretraining/code/extend_tokenizer_utils.py +++ b/tutorials/llm/llama/domain-adaptive-pretraining/code/extend_tokenizer_utils.py @@ -1,3 +1,17 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import glob import io import json diff --git a/tutorials/llm/llama/domain-adaptive-pretraining/code/get_high_freq_tokens.py b/tutorials/llm/llama/domain-adaptive-pretraining/code/get_high_freq_tokens.py index 0c8401cb69d2..613f72d7eba1 100755 --- a/tutorials/llm/llama/domain-adaptive-pretraining/code/get_high_freq_tokens.py +++ b/tutorials/llm/llama/domain-adaptive-pretraining/code/get_high_freq_tokens.py @@ -1,3 +1,17 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import json import sys from bisect import bisect_left diff --git a/tutorials/llm/llama/domain-adaptive-pretraining/code/tokenization_helper.py b/tutorials/llm/llama/domain-adaptive-pretraining/code/tokenization_helper.py index e7a0c29846d7..3c1d221ba07a 100755 --- a/tutorials/llm/llama/domain-adaptive-pretraining/code/tokenization_helper.py +++ b/tutorials/llm/llama/domain-adaptive-pretraining/code/tokenization_helper.py @@ -1,3 +1,17 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import glob import io import json